You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by le...@apache.org on 2022/09/16 07:37:11 UTC
[incubator-linkis] branch dev-1.3.0 updated: [Feature] support linkis on ldh (#3431)
This is an automated email from the ASF dual-hosted git repository.
legendtkl pushed a commit to branch dev-1.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.3.0 by this push:
new 4d0d81496 [Feature] support linkis on ldh (#3431)
4d0d81496 is described below
commit 4d0d81496718878f1f50153411a428ed4a7cb633
Author: Aaron Lin <51...@users.noreply.github.com>
AuthorDate: Fri Sep 16 15:37:04 2022 +0800
[Feature] support linkis on ldh (#3431)
---
linkis-dist/docker/scripts/entry-point-ldh.sh | 2 +
.../linkis/templates/configmap-linkis-config.yaml | 59 ++-----
.../templates/linkis-cg-engineconnmanager.yaml | 15 ++
.../linkis/templates/linkis-cg-engineplugin.yaml | 33 ++++
.../linkis/templates/linkis-cg-entrance.yaml | 15 ++
.../helm/charts/linkis/templates/linkis-ps-cs.yaml | 184 --------------------
.../templates/linkis-ps-data-source-manager.yaml | 184 --------------------
.../linkis/templates/linkis-ps-metadataquery.yaml | 187 ---------------------
.../linkis/templates/linkis-ps-publicservice.yaml | 24 +++
linkis-dist/helm/charts/linkis/values.yaml | 60 ++-----
.../{install-ldh.sh => install-charts-with-ldh.sh} | 19 +--
linkis-dist/helm/scripts/install-charts.sh | 23 +--
linkis-dist/helm/scripts/install-ldh.sh | 7 +-
.../{install-charts.sh => install-linkis.sh} | 4 +-
.../resources/ldh/configmaps/configmap-hadoop.yaml | 6 +
.../resources/ldh/configmaps/configmap-spark.yaml | 34 ----
linkis-dist/helm/scripts/resources/ldh/ldh.yaml | 92 +++-------
17 files changed, 162 insertions(+), 786 deletions(-)
diff --git a/linkis-dist/docker/scripts/entry-point-ldh.sh b/linkis-dist/docker/scripts/entry-point-ldh.sh
index 252f34498..1e03b3017 100644
--- a/linkis-dist/docker/scripts/entry-point-ldh.sh
+++ b/linkis-dist/docker/scripts/entry-point-ldh.sh
@@ -34,6 +34,8 @@ hdfs dfs -mkdir -p /spark2-history
hdfs dfs -chmod -R 777 /spark2-history
hdfs dfs -mkdir -p /completed-jobs
hdfs dfs -chmod -R 777 /completed-jobs
+hdfs dfs -chmod 777 /
+
# - hive
/opt/ldh/current/hive/bin/schematool -initSchema -dbType mysql
diff --git a/linkis-dist/helm/charts/linkis/templates/configmap-linkis-config.yaml b/linkis-dist/helm/charts/linkis/templates/configmap-linkis-config.yaml
index 843bf3f61..53c64f647 100644
--- a/linkis-dist/helm/charts/linkis/templates/configmap-linkis-config.yaml
+++ b/linkis-dist/helm/charts/linkis/templates/configmap-linkis-config.yaml
@@ -257,12 +257,16 @@ data:
spring.server.port={{ .Values.cgEntrance.port }}
linkis-ps-publicservice.properties: |
+ wds.linkis.server.mdm.service.instance.expire-in-seconds=1800
+
##restful
- wds.linkis.server.restful.scan.packages=org.apache.linkis.jobhistory.restful,org.apache.linkis.variable.restful,org.apache.linkis.configuration.restful,org.apache.linkis.udf.api,org.apache.linkis.filesystem.restful,org.apache.linkis.filesystem.restful,org.apache.linkis.instance.label.restful,org.apache.linkis.metadata.restful.api,org.apache.linkis.cs.server.restful,org.apache.linkis.bml.restful,org.apache.linkis.errorcode.server.restful
+ wds.linkis.server.restful.scan.packages=org.apache.linkis.cs.server.restful,org.apache.linkis.datasourcemanager.core.restful,org.apache.linkis.metadata.query.server.restful,org.apache.linkis.jobhistory.restful,org.apache.linkis.variable.restful,org.apache.linkis.configuration.restful,org.apache.linkis.udf.api,org.apache.linkis.filesystem.restful,org.apache.linkis.filesystem.restful,org.apache.linkis.instance.label.restful,org.apache.linkis.metadata.restful.api,org.apache.linkis.cs.se [...]
+
##mybatis
- wds.linkis.server.mybatis.mapperLocations=classpath:org/apache/linkis/jobhistory/dao/impl/*.xml,classpath:org/apache/linkis/variable/dao/impl/*.xml,classpath:org/apache/linkis/configuration/dao/impl/*.xml,classpath:org/apache/linkis/udf/dao/impl/*.xml,classpath:org/apache/linkis/instance/label/dao/impl/*.xml,classpath:org/apache/linkis/metadata/hive/dao/impl/*.xml,org/apache/linkis/metadata/dao/impl/*.xml,classpath:org/apache/linkis/bml/dao/impl/*.xml
- wds.linkis.server.mybatis.typeAliasesPackage=org.apache.linkis.configuration.entity,org.apache.linkis.jobhistory.entity,org.apache.linkis.udf.entity,org.apache.linkis.variable.entity,org.apache.linkis.instance.label.entity,org.apache.linkis.manager.entity,org.apache.linkis.metadata.domain,org.apache.linkis.bml.Entity
- wds.linkis.server.mybatis.BasePackage=org.apache.linkis.jobhistory.dao,org.apache.linkis.variable.dao,org.apache.linkis.configuration.dao,org.apache.linkis.udf.dao,org.apache.linkis.instance.label.dao,org.apache.linkis.metadata.hive.dao,org.apache.linkis.metadata.dao,org.apache.linkis.bml.dao,org.apache.linkis.errorcode.server.dao,org.apache.linkis.publicservice.common.lock.dao
+ wds.linkis.server.mybatis.mapperLocations=classpath*:org/apache/linkis/cs/persistence/dao/impl/*.xml,classpath:org/apache/linkis/datasourcemanager/core/dao/mapper/*.xml,classpath:org/apache/linkis/jobhistory/dao/impl/*.xml,classpath:org/apache/linkis/variable/dao/impl/*.xml,classpath:org/apache/linkis/configuration/dao/impl/*.xml,classpath:org/apache/linkis/udf/dao/impl/*.xml,classpath:org/apache/linkis/instance/label/dao/impl/*.xml,classpath:org/apache/linkis/metadata/hive/dao/impl/ [...]
+ wds.linkis.server.mybatis.typeAliasesPackage=org.apache.linkis.cs.persistence.entity,org.apache.linkis.datasourcemanager.common.domain,org.apache.linkis.datasourcemanager.core.vo,org.apache.linkis.configuration.entity,org.apache.linkis.jobhistory.entity,org.apache.linkis.udf.entity,org.apache.linkis.variable.entity,org.apache.linkis.instance.label.entity,org.apache.linkis.manager.entity,org.apache.linkis.metadata.domain,org.apache.linkis.bml.entity
+ wds.linkis.server.mybatis.BasePackage=org.apache.linkis.cs.persistence.dao,org.apache.linkis.datasourcemanager.core.dao,org.apache.linkis.jobhistory.dao,org.apache.linkis.variable.dao,org.apache.linkis.configuration.dao,org.apache.linkis.udf.dao,org.apache.linkis.instance.label.dao,org.apache.linkis.metadata.hive.dao,org.apache.linkis.metadata.dao,org.apache.linkis.bml.dao,org.apache.linkis.errorcode.server.dao,org.apache.linkis.publicservice.common.lock.dao
+
##workspace
wds.linkis.workspace.filesystem.hdfsuserrootpath.suffix=/
wds.linkis.server.component.exclude.classes=org.apache.linkis.entranceclient.conf.ClientForEntranceSpringConfiguration,org.apache.linkis.entranceclient.conf.ClientSpringConfiguration,org.apache.linkis.entrance.conf.EntranceSpringConfiguration
@@ -271,51 +275,20 @@ data:
hive.meta.url={{ .Values.linkis.deps.hive.meta.url }}
hive.meta.user={{ .Values.linkis.deps.hive.meta.user }}
hive.meta.password={{ .Values.linkis.deps.hive.meta.password }}
+ wds.linkis.metadata.hive.encode.enabled=false
# associated with the logged-in user when querying metadata:default value is true
-
#linkis.metadata.hive.permission.with-login-user-enabled
- ##Spring
- spring.server.port={{ .Values.psPublicService.port }}
- spring.spring.main.allow-bean-definition-overriding=true
-
- linkis-ps-cs.properties: |
- ##restful
- wds.linkis.server.restful.scan.packages=org.apache.linkis.cs.server.restful
- ##mybatis
- wds.linkis.server.mybatis.mapperLocations=classpath*:org/apache/linkis/cs/persistence/dao/impl/*.xml
- wds.linkis.server.mybatis.typeAliasesPackage=org.apache.linkis.cs.persistence.entity
- wds.linkis.server.mybatis.BasePackage=org.apache.linkis.cs.persistence.dao
- ##Spring
- spring.server.port={{ .Values.psCs.port }}
- # ps-cs prefix must be started with 'cs_'
- spring.eureka.instance.metadata-map.route=cs_1_dev
+ #wds.linkis.jobhistory.undone.job.minimum.id=0
+ #wds.linkis.jobhistory.undone.job.refreshtime.daily=00:15
wds.linkis.cs.deserialize.replace_package_header.enable=false
- linkis-ps-data-source-manager.properties: |
- ##restful
- wds.linkis.server.restful.scan.packages=org.apache.linkis.datasourcemanager.core.restful
- ##mybatis
- wds.linkis.server.mybatis.mapperLocations=classpath:org/apache/linkis/datasourcemanager/core/dao/mapper/*.xml
- wds.linkis.server.mybatis.typeAliasesPackage=org.apache.linkis.datasourcemanager.common.domain,org.apache.linkis.datasourcemanager.core.vo
- wds.linkis.server.mybatis.BasePackage=org.apache.linkis.datasourcemanager.core.dao
-
- ##hive meta
- wds.linkis.metadata.hive.encode.enabled=false
- hive.meta.url={{ .Values.linkis.deps.hive.meta.url }}
- hive.meta.user={{ .Values.linkis.deps.hive.meta.user }}
- hive.meta.password={{ .Values.linkis.deps.hive.meta.password }}
+ wds.linkis.rpc.conf.enable.local.message=true
+ wds.linkis.rpc.conf.local.app.list=linkis-ps-publicservice
##Spring
- spring.server.port={{ .Values.psDataSourceManager.port }}
+ spring.server.port={{ .Values.psPublicService.port }}
spring.spring.main.allow-bean-definition-overriding=true
spring.spring.jackson.serialization.FAIL_ON_EMPTY_BEANS=false
- spring.jackson.serialization.FAIL_ON_EMPTY_BEANS=false
-
- linkis-ps-metadataquery.properties: |
- wds.linkis.server.mdm.service.instance.expire-in-seconds=1800
- wds.linkis.server.restful.scan.packages=org.apache.linkis.metadatamanager.server.restful
- wds.linkis.server.dsm.app.name=linkis-ps-data-source-manager
-
- ##Spring
- spring.server.port={{ .Values.psMetadataQuery.port }}
+ # ps-cs prefix must be started with 'cs_'
+ spring.eureka.instance.metadata-map.route=cs_1_dev
diff --git a/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineconnmanager.yaml b/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineconnmanager.yaml
index b04b65d8e..8e89cb584 100644
--- a/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineconnmanager.yaml
+++ b/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineconnmanager.yaml
@@ -106,6 +106,13 @@ spec:
{{- else }}
emptyDir: {}
{{- end }}
+ {{- if eq .Values.linkis.featureGates.localMode false }}
+ - name: hadoop-conf
+ projected:
+ sources:
+ - configMap:
+ name: {{ .Values.linkis.deps.hadoop.configMapName }}
+ {{- end }}
serviceAccountName: {{ include "linkis.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.cgEngineConnManager.podSecurityContext | nindent 8 }}
@@ -142,6 +149,10 @@ spec:
- name: DEBUG_PORT
value: "5005"
{{- end }}
+ {{- if eq .Values.linkis.featureGates.localMode false }}
+ - name: HADOOP_CONF_DIR
+ value: /etc/hadoop-conf
+ {{- end }}
- name: SERVER_HEAP_SIZE
value: {{ .Values.cgEngineConnManager.jvmHeapSize }}
- name: EUREKA_URL
@@ -162,6 +173,10 @@ spec:
{{ toYaml .Values.cgEngineConnManager.envs.froms | indent 12 }}
{{- end }}
volumeMounts:
+ {{- if eq .Values.linkis.featureGates.localMode false }}
+ - name: hadoop-conf
+ mountPath: /etc/hadoop-conf
+ {{- end }}
- name: conf
mountPath: {{ .Values.linkis.locations.confDir }}
- name: log
diff --git a/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineplugin.yaml b/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineplugin.yaml
index 8e2177dce..d97d6b107 100644
--- a/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineplugin.yaml
+++ b/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineplugin.yaml
@@ -106,6 +106,23 @@ spec:
{{- else }}
emptyDir: {}
{{- end }}
+ {{- if eq .Values.linkis.featureGates.localMode false }}
+ - name: hadoop-conf
+ projected:
+ sources:
+ - configMap:
+ name: {{ .Values.linkis.deps.hadoop.configMapName }}
+ - name: hive-conf
+ projected:
+ sources:
+ - configMap:
+ name: {{ .Values.linkis.deps.hive.configMapName }}
+ - name: spark-conf
+ projected:
+ sources:
+ - configMap:
+ name: {{ .Values.linkis.deps.spark.configMapName }}
+ {{- end }}
serviceAccountName: {{ include "linkis.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.cgEnginePlugin.podSecurityContext | nindent 8 }}
@@ -142,6 +159,14 @@ spec:
- name: DEBUG_PORT
value: "5005"
{{- end }}
+ {{- if eq .Values.linkis.featureGates.localMode false }}
+ - name: HADOOP_CONF_DIR
+ value: /etc/hadoop-conf
+ - name: HIVE_CONF_DIR
+ value: /etc/hive-conf
+ - name: SPARK_CONF_DIR
+ value: /etc/spark-conf
+ {{- end }}
- name: SERVER_HEAP_SIZE
value: {{ .Values.cgEnginePlugin.jvmHeapSize }}
- name: EUREKA_URL
@@ -162,6 +187,14 @@ spec:
{{ toYaml .Values.cgEnginePlugin.envs.froms | indent 12 }}
{{- end }}
volumeMounts:
+ {{- if eq .Values.linkis.featureGates.localMode false }}
+ - name: hadoop-conf
+ mountPath: /etc/hadoop-conf
+ - name: hive-conf
+ mountPath: /etc/hive-conf
+ - name: spark-conf
+ mountPath: /etc/spark-conf
+ {{- end }}
- name: conf
mountPath: {{ .Values.linkis.locations.confDir }}
- name: log
diff --git a/linkis-dist/helm/charts/linkis/templates/linkis-cg-entrance.yaml b/linkis-dist/helm/charts/linkis/templates/linkis-cg-entrance.yaml
index 727a37d72..7b3e9d16f 100644
--- a/linkis-dist/helm/charts/linkis/templates/linkis-cg-entrance.yaml
+++ b/linkis-dist/helm/charts/linkis/templates/linkis-cg-entrance.yaml
@@ -106,6 +106,13 @@ spec:
{{- else }}
emptyDir: {}
{{- end }}
+ {{- if eq .Values.linkis.featureGates.localMode false }}
+ - name: hadoop-conf
+ projected:
+ sources:
+ - configMap:
+ name: {{ .Values.linkis.deps.hadoop.configMapName }}
+ {{- end }}
serviceAccountName: {{ include "linkis.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.cgEntrance.podSecurityContext | nindent 8 }}
@@ -142,6 +149,10 @@ spec:
- name: DEBUG_PORT
value: "5005"
{{- end }}
+ {{- if eq .Values.linkis.featureGates.localMode false }}
+ - name: HADOOP_CONF_DIR
+ value: /etc/hadoop-conf
+ {{- end }}
- name: SERVER_HEAP_SIZE
value: {{ .Values.cgEntrance.jvmHeapSize }}
- name: EUREKA_URL
@@ -162,6 +173,10 @@ spec:
{{ toYaml .Values.cgEntrance.envs.froms | indent 12 }}
{{- end }}
volumeMounts:
+ {{- if eq .Values.linkis.featureGates.localMode false }}
+ - name: hadoop-conf
+ mountPath: /etc/hadoop-conf
+ {{- end }}
- name: conf
mountPath: {{ .Values.linkis.locations.confDir }}
- name: log
diff --git a/linkis-dist/helm/charts/linkis/templates/linkis-ps-cs.yaml b/linkis-dist/helm/charts/linkis/templates/linkis-ps-cs.yaml
deleted file mode 100644
index 994d3c022..000000000
--- a/linkis-dist/helm/charts/linkis/templates/linkis-ps-cs.yaml
+++ /dev/null
@@ -1,184 +0,0 @@
----
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: v1
-kind: Service
-metadata:
- name: {{ include "linkis.fullname" . }}-ps-cs
- labels:
- app: {{ include "linkis.fullname" . }}-ps-cs
- {{- include "linkis.psCs.labels" . | nindent 4 }}
- annotations:
- prometheus.io/path: {{ .Values.psCs.prometheus.metricsPath }}
- prometheus.io/port: '{{ .Values.psCs.port }}'
- prometheus.io/scrape: 'true'
-spec:
- ports:
- - name: "http"
- protocol: TCP
- port: {{ .Values.psCs.port }}
- selector:
- {{- include "linkis.psCs.selectorLabels" . | nindent 4 }}
- app: {{ include "linkis.fullname" . }}-ps-cs
- {{- toYaml .Values.svc | nindent 2 }}
----
-apiVersion: v1
-kind: Service
-metadata:
- name: {{ include "linkis.fullname" . }}-ps-cs-headless
- labels:
- app: {{ include "linkis.fullname" . }}-ps-cs
- {{- include "linkis.psCs.labels" . | nindent 4 }}
- annotations:
- prometheus.io/path: {{ .Values.psCs.prometheus.metricsPath }}
- prometheus.io/port: '{{ .Values.psCs.port }}'
- prometheus.io/scrape: 'true'
-spec:
- ports:
- - name: "http"
- protocol: TCP
- port: {{ .Values.psCs.port }}
- selector:
- {{- include "linkis.psCs.selectorLabels" . | nindent 4 }}
- app: {{ include "linkis.fullname" . }}-ps-cs
- clusterIP: None
- type: ClusterIP
- publishNotReadyAddresses: true
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: {{ include "linkis.fullname" . }}-ps-cs
- labels:
- app: {{ include "linkis.fullname" . }}-ps-cs
- version: {{ .Chart.AppVersion }}
- {{- include "linkis.psCs.labels" . | nindent 4 }}
- {{- if .Values.psCs.annotations }}
- annotations:
- {{- toYaml .Values.psCs.annotations | nindent 4 }}
- {{- end }}
-spec:
- replicas: {{ .Values.psCs.replicas }}
- selector:
- matchLabels:
- {{- include "linkis.psCs.selectorLabels" . | nindent 6 }}
- app: {{ include "linkis.fullname" . }}-ps-cs
- template:
- metadata:
- {{- with .Values.psCs.annotations }}
- annotations:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- labels:
- app: {{ include "linkis.fullname" . }}-ps-cs
- version: {{ .Chart.AppVersion }}
- {{- include "linkis.psCs.selectorLabels" . | nindent 8 }}
- spec:
- subdomain: {{ include "linkis.fullname" . }}-ps-cs-headless
- {{- with .Values.imagePullSecrets }}
- imagePullSecrets:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- volumes:
- - name: conf
- configMap:
- name: {{ include "linkis.fullname" . }}-linkis-config
- - name: log
- emptyDir: {}
- - name: runtime
- {{- if .Values.linkis.featureGates.localMode }}
- hostPath:
- path: {{ .Values.linkis.locations.hostPath }}
- type: DirectoryOrCreate
- {{- else }}
- emptyDir: {}
- {{- end }}
- serviceAccountName: {{ include "linkis.serviceAccountName" . }}
- securityContext:
- {{- toYaml .Values.psCs.podSecurityContext | nindent 8 }}
- containers:
- - name: "cs"
- securityContext:
- {{- toYaml .Values.psCs.securityContext | nindent 12 }}
- image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
- imagePullPolicy: {{ .Values.image.pullPolicy }}
- command:
- - /bin/bash
- - -ecx
- - >-
- RUN_IN_FOREGROUND=true {{ .Values.linkis.locations.homeDir }}/sbin/linkis-daemon.sh start ps-cs
- ports:
- - name: "http"
- containerPort: {{ .Values.psCs.port }}
- protocol: TCP
- {{- if .Values.linkis.featureGates.enableJvmRemoteDebug }}
- - name: "debug"
- containerPort: 5005
- protocol: TCP
- {{- end }}
- # TODO: replace with httpGet when spring-boot readiness probe is implemented.
- readinessProbe:
- initialDelaySeconds: 15
- periodSeconds: 5
- timeoutSeconds: 20
- failureThreshold: 10
- tcpSocket:
- port: {{ .Values.psCs.port }}
- env:
- {{- if .Values.linkis.featureGates.enableJvmRemoteDebug }}
- - name: DEBUG_PORT
- value: "5005"
- {{- end }}
- - name: SERVER_HEAP_SIZE
- value: {{ .Values.psCs.jvmHeapSize }}
- - name: EUREKA_URL
- value: {{- include "linkis.registration.url" . | quote | indent 1 }}
- - name: EUREKA_PREFER_IP
- value: "true"
- - name: EUREKA_PORT
- value: "{{ .Values.mgEureka.port }}"
- - name: POD_NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- {{- if .Values.psCs.envs.extras }}
-{{ toYaml .Values.psCs.envs.extras | indent 12 }}
- {{- end }}
- {{- if .Values.psCs.envs.froms }}
- envFrom:
-{{ toYaml .Values.psCs.envs.froms | indent 12 }}
- {{- end }}
- volumeMounts:
- - name: conf
- mountPath: {{ .Values.linkis.locations.confDir }}
- - name: log
- mountPath: {{ .Values.linkis.locations.logDir }}
- - name: runtime
- mountPath: {{ .Values.linkis.locations.runtimeDir }}
- resources:
- {{- toYaml .Values.psCs.resources | nindent 12 }}
- {{- with .Values.psCs.nodeSelector }}
- nodeSelector:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- {{- with .Values.psCs.affinity }}
- affinity:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- {{- with .Values.psCs.tolerations }}
- tolerations:
- {{- toYaml . | nindent 8 }}
- {{- end }}
diff --git a/linkis-dist/helm/charts/linkis/templates/linkis-ps-data-source-manager.yaml b/linkis-dist/helm/charts/linkis/templates/linkis-ps-data-source-manager.yaml
deleted file mode 100644
index 0e3c6469e..000000000
--- a/linkis-dist/helm/charts/linkis/templates/linkis-ps-data-source-manager.yaml
+++ /dev/null
@@ -1,184 +0,0 @@
----
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: v1
-kind: Service
-metadata:
- name: {{ include "linkis.fullname" . }}-ps-data-source-manager
- labels:
- app: {{ include "linkis.fullname" . }}-ps-data-source-manager
- {{- include "linkis.psDataSourceManager.labels" . | nindent 4 }}
- annotations:
- prometheus.io/path: {{ .Values.psDataSourceManager.prometheus.metricsPath }}
- prometheus.io/port: '{{ .Values.psDataSourceManager.port }}'
- prometheus.io/scrape: 'true'
-spec:
- ports:
- - name: "http"
- protocol: TCP
- port: {{ .Values.psDataSourceManager.port }}
- selector:
- {{- include "linkis.psDataSourceManager.selectorLabels" . | nindent 4 }}
- app: {{ include "linkis.fullname" . }}-ps-data-source-manager
- {{- toYaml .Values.svc | nindent 2 }}
----
-apiVersion: v1
-kind: Service
-metadata:
- name: {{ include "linkis.fullname" . }}-ps-data-source-manager-headless
- labels:
- app: {{ include "linkis.fullname" . }}-ps-data-source-manager
- {{- include "linkis.psDataSourceManager.labels" . | nindent 4 }}
- annotations:
- prometheus.io/path: {{ .Values.psDataSourceManager.prometheus.metricsPath }}
- prometheus.io/port: '{{ .Values.psDataSourceManager.port }}'
- prometheus.io/scrape: 'true'
-spec:
- ports:
- - name: "http"
- protocol: TCP
- port: {{ .Values.psDataSourceManager.port }}
- selector:
- {{- include "linkis.psDataSourceManager.selectorLabels" . | nindent 4 }}
- app: {{ include "linkis.fullname" . }}-ps-data-source-manager
- clusterIP: None
- type: ClusterIP
- publishNotReadyAddresses: true
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: {{ include "linkis.fullname" . }}-ps-data-source-manager
- labels:
- app: {{ include "linkis.fullname" . }}-ps-data-source-manager
- version: {{ .Chart.AppVersion }}
- {{- include "linkis.psDataSourceManager.labels" . | nindent 4 }}
- {{- if .Values.psDataSourceManager.annotations }}
- annotations:
- {{- toYaml .Values.psDataSourceManager.annotations | nindent 4 }}
- {{- end }}
-spec:
- replicas: {{ .Values.psDataSourceManager.replicas }}
- selector:
- matchLabels:
- {{- include "linkis.psDataSourceManager.selectorLabels" . | nindent 6 }}
- app: {{ include "linkis.fullname" . }}-ps-data-source-manager
- template:
- metadata:
- {{- with .Values.psDataSourceManager.annotations }}
- annotations:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- labels:
- app: {{ include "linkis.fullname" . }}-ps-data-source-manager
- version: {{ .Chart.AppVersion }}
- {{- include "linkis.psDataSourceManager.selectorLabels" . | nindent 8 }}
- spec:
- subdomain: {{ include "linkis.fullname" . }}-ps-data-source-manager-headless
- {{- with .Values.imagePullSecrets }}
- imagePullSecrets:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- volumes:
- - name: conf
- configMap:
- name: {{ include "linkis.fullname" . }}-linkis-config
- - name: log
- emptyDir: {}
- - name: runtime
- {{- if .Values.linkis.featureGates.localMode }}
- hostPath:
- path: {{ .Values.linkis.locations.hostPath }}
- type: DirectoryOrCreate
- {{- else }}
- emptyDir: {}
- {{- end }}
- serviceAccountName: {{ include "linkis.serviceAccountName" . }}
- securityContext:
- {{- toYaml .Values.psDataSourceManager.podSecurityContext | nindent 8 }}
- containers:
- - name: "data-source-manager"
- securityContext:
- {{- toYaml .Values.psDataSourceManager.securityContext | nindent 12 }}
- image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
- imagePullPolicy: {{ .Values.image.pullPolicy }}
- command:
- - /bin/bash
- - -ecx
- - >-
- RUN_IN_FOREGROUND=true {{ .Values.linkis.locations.homeDir }}/sbin/linkis-daemon.sh start ps-data-source-manager
- ports:
- - name: "http"
- containerPort: {{ .Values.psDataSourceManager.port }}
- protocol: TCP
- {{- if .Values.linkis.featureGates.enableJvmRemoteDebug }}
- - name: "debug"
- containerPort: 5005
- protocol: TCP
- {{- end }}
- # TODO: replace with httpGet when spring-boot readiness probe is implemented.
- readinessProbe:
- initialDelaySeconds: 15
- periodSeconds: 5
- timeoutSeconds: 20
- failureThreshold: 10
- tcpSocket:
- port: {{ .Values.psDataSourceManager.port }}
- env:
- {{- if .Values.linkis.featureGates.enableJvmRemoteDebug }}
- - name: DEBUG_PORT
- value: "5005"
- {{- end }}
- - name: SERVER_HEAP_SIZE
- value: {{ .Values.psDataSourceManager.jvmHeapSize }}
- - name: EUREKA_URL
- value: {{- include "linkis.registration.url" . | quote | indent 1 }}
- - name: EUREKA_PREFER_IP
- value: "true"
- - name: EUREKA_PORT
- value: "{{ .Values.mgEureka.port }}"
- - name: POD_NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- {{- if .Values.psDataSourceManager.envs.extras }}
-{{ toYaml .Values.psDataSourceManager.envs.extras | indent 12 }}
- {{- end }}
- {{- if .Values.psDataSourceManager.envs.froms }}
- envFrom:
-{{ toYaml .Values.psDataSourceManager.envs.froms | indent 12 }}
- {{- end }}
- volumeMounts:
- - name: conf
- mountPath: {{ .Values.linkis.locations.confDir }}
- - name: log
- mountPath: {{ .Values.linkis.locations.logDir }}
- - name: runtime
- mountPath: {{ .Values.linkis.locations.runtimeDir }}
- resources:
- {{- toYaml .Values.psDataSourceManager.resources | nindent 12 }}
- {{- with .Values.psDataSourceManager.nodeSelector }}
- nodeSelector:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- {{- with .Values.psDataSourceManager.affinity }}
- affinity:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- {{- with .Values.psDataSourceManager.tolerations }}
- tolerations:
- {{- toYaml . | nindent 8 }}
- {{- end }}
diff --git a/linkis-dist/helm/charts/linkis/templates/linkis-ps-metadataquery.yaml b/linkis-dist/helm/charts/linkis/templates/linkis-ps-metadataquery.yaml
deleted file mode 100644
index 4da8dd257..000000000
--- a/linkis-dist/helm/charts/linkis/templates/linkis-ps-metadataquery.yaml
+++ /dev/null
@@ -1,187 +0,0 @@
----
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{{- if .Values.linkis.featureGates.enableMetadataQuery }}
-apiVersion: v1
-kind: Service
-metadata:
- name: {{ include "linkis.fullname" . }}-ps-metadataquery
- labels:
- app: {{ include "linkis.fullname" . }}-ps-metadataquery
- {{- include "linkis.psMetadataQuery.labels" . | nindent 4 }}
- annotations:
- prometheus.io/path: {{ .Values.psMetadataQuery.prometheus.metricsPath }}
- prometheus.io/port: '{{ .Values.psMetadataQuery.port }}'
- prometheus.io/scrape: 'true'
-spec:
- ports:
- - name: "http"
- protocol: TCP
- port: {{ .Values.psMetadataQuery.port }}
- selector:
- {{- include "linkis.psMetadataQuery.selectorLabels" . | nindent 4 }}
- app: {{ include "linkis.fullname" . }}-ps-metadataquery
- {{- toYaml .Values.svc | nindent 2 }}
----
-apiVersion: v1
-kind: Service
-metadata:
- name: {{ include "linkis.fullname" . }}-ps-metadataquery-headless
- labels:
- app: {{ include "linkis.fullname" . }}-ps-metadataquery
- {{- include "linkis.psMetadataQuery.labels" . | nindent 4 }}
- annotations:
- prometheus.io/path: {{ .Values.psMetadataQuery.prometheus.metricsPath }}
- prometheus.io/port: '{{ .Values.psMetadataQuery.port }}'
- prometheus.io/scrape: 'true'
-spec:
- ports:
- - name: "http"
- protocol: TCP
- port: {{ .Values.psMetadataQuery.port }}
- selector:
- {{- include "linkis.psMetadataQuery.selectorLabels" . | nindent 4 }}
- app: {{ include "linkis.fullname" . }}-ps-metadataquery
- clusterIP: None
- type: ClusterIP
- publishNotReadyAddresses: true
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: {{ include "linkis.fullname" . }}-ps-metadataquery
- labels:
- app: {{ include "linkis.fullname" . }}-ps-metadataquery
- version: {{ .Chart.AppVersion }}
- {{- include "linkis.psMetadataQuery.labels" . | nindent 4 }}
- {{- if .Values.psMetadataQuery.annotations }}
- annotations:
- {{- toYaml .Values.psMetadataQuery.annotations | nindent 4 }}
- {{- end }}
-spec:
- replicas: {{ .Values.psMetadataQuery.replicas }}
- selector:
- matchLabels:
- {{- include "linkis.psMetadataQuery.selectorLabels" . | nindent 6 }}
- app: {{ include "linkis.fullname" . }}-ps-metadataquery
- template:
- metadata:
- {{- with .Values.psMetadataQuery.annotations }}
- annotations:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- labels:
- app: {{ include "linkis.fullname" . }}-ps-metadataquery
- version: {{ .Chart.AppVersion }}
- {{- include "linkis.psMetadataQuery.selectorLabels" . | nindent 8 }}
- spec:
- subdomain: {{ include "linkis.fullname" . }}-ps-metadataquery-headless
- {{- with .Values.imagePullSecrets }}
- imagePullSecrets:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- volumes:
- - name: conf
- configMap:
- name: {{ include "linkis.fullname" . }}-linkis-config
- - name: log
- emptyDir: {}
- - name: runtime
- {{- if .Values.linkis.featureGates.localMode }}
- hostPath:
- path: {{ .Values.linkis.locations.hostPath }}
- type: DirectoryOrCreate
- {{- else }}
- emptyDir: {}
- {{- end }}
- serviceAccountName: {{ include "linkis.serviceAccountName" . }}
- securityContext:
- {{- toYaml .Values.psMetadataQuery.podSecurityContext | nindent 8 }}
- containers:
- - name: "metadataquery"
- securityContext:
- {{- toYaml .Values.psMetadataQuery.securityContext | nindent 12 }}
- image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
- imagePullPolicy: {{ .Values.image.pullPolicy }}
- command:
- - /bin/bash
- - -ecx
- - >-
- RUN_IN_FOREGROUND=true {{ .Values.linkis.locations.homeDir }}/sbin/linkis-daemon.sh start ps-metadataquery
- ports:
- - name: "http"
- containerPort: {{ .Values.psMetadataQuery.port }}
- protocol: TCP
- {{- if .Values.linkis.featureGates.enableJvmRemoteDebug }}
- - name: "debug"
- containerPort: 5005
- protocol: TCP
- {{- end }}
- # TODO: replace with httpGet when spring-boot readiness probe is implemented.
- readinessProbe:
- initialDelaySeconds: 15
- periodSeconds: 5
- timeoutSeconds: 20
- failureThreshold: 10
- tcpSocket:
- port: {{ .Values.psMetadataQuery.port }}
- env:
- {{- if .Values.linkis.featureGates.enableJvmRemoteDebug }}
- - name: DEBUG_PORT
- value: "5005"
- {{- end }}
- - name: SERVER_HEAP_SIZE
- value: {{ .Values.psMetadataQuery.jvmHeapSize }}
- - name: EUREKA_URL
- value: {{- include "linkis.registration.url" . | quote | indent 1 }}
- - name: EUREKA_PREFER_IP
- value: "true"
- - name: EUREKA_PORT
- value: "{{ .Values.mgEureka.port }}"
- - name: POD_NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- {{- if .Values.psMetadataQuery.envs.extras }}
-{{ toYaml .Values.psMetadataQuery.envs.extras | indent 12 }}
- {{- end }}
- {{- if .Values.psMetadataQuery.envs.froms }}
- envFrom:
-{{ toYaml .Values.psMetadataQuery.envs.froms | indent 12 }}
- {{- end }}
- volumeMounts:
- - name: conf
- mountPath: {{ .Values.linkis.locations.confDir }}
- - name: log
- mountPath: {{ .Values.linkis.locations.logDir }}
- - name: runtime
- mountPath: {{ .Values.linkis.locations.runtimeDir }}
- resources:
- {{- toYaml .Values.psMetadataQuery.resources | nindent 12 }}
- {{- with .Values.psMetadataQuery.nodeSelector }}
- nodeSelector:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- {{- with .Values.psMetadataQuery.affinity }}
- affinity:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- {{- with .Values.psMetadataQuery.tolerations }}
- tolerations:
- {{- toYaml . | nindent 8 }}
- {{- end }}
-
-{{- end }}
diff --git a/linkis-dist/helm/charts/linkis/templates/linkis-ps-publicservice.yaml b/linkis-dist/helm/charts/linkis/templates/linkis-ps-publicservice.yaml
index a19457ab7..c219c6161 100644
--- a/linkis-dist/helm/charts/linkis/templates/linkis-ps-publicservice.yaml
+++ b/linkis-dist/helm/charts/linkis/templates/linkis-ps-publicservice.yaml
@@ -106,6 +106,18 @@ spec:
{{- else }}
emptyDir: {}
{{- end }}
+ {{- if eq .Values.linkis.featureGates.localMode false }}
+ - name: hadoop-conf
+ projected:
+ sources:
+ - configMap:
+ name: {{ .Values.linkis.deps.hadoop.configMapName }}
+ - name: hive-conf
+ projected:
+ sources:
+ - configMap:
+ name: {{ .Values.linkis.deps.hive.configMapName }}
+ {{- end }}
serviceAccountName: {{ include "linkis.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.psPublicService.podSecurityContext | nindent 8 }}
@@ -142,6 +154,12 @@ spec:
- name: DEBUG_PORT
value: "5005"
{{- end }}
+ {{- if eq .Values.linkis.featureGates.localMode false }}
+ - name: HADOOP_CONF_DIR
+ value: /etc/hadoop-conf
+ - name: HIVE_CONF_DIR
+ value: /etc/hive-conf
+ {{- end }}
- name: SERVER_HEAP_SIZE
value: {{ .Values.psPublicService.jvmHeapSize }}
- name: EUREKA_URL
@@ -162,6 +180,12 @@ spec:
{{ toYaml .Values.psPublicService.envs.froms | indent 12 }}
{{- end }}
volumeMounts:
+ {{- if eq .Values.linkis.featureGates.localMode false }}
+ - name: hadoop-conf
+ mountPath: /etc/hadoop-conf
+ - name: hive-conf
+ mountPath: /etc/hive-conf
+ {{- end }}
- name: conf
mountPath: {{ .Values.linkis.locations.confDir }}
- name: log
diff --git a/linkis-dist/helm/charts/linkis/values.yaml b/linkis-dist/helm/charts/linkis/values.yaml
index d60784b05..16fa1fce9 100644
--- a/linkis-dist/helm/charts/linkis/values.yaml
+++ b/linkis-dist/helm/charts/linkis/values.yaml
@@ -110,6 +110,7 @@ linkis:
version: 2.7
hadoop:
version: 2.7.2
+ configMapName: hadoop-conf
yarn:
restfulUrl: http://localhost:8080
authEnable: false
@@ -121,12 +122,20 @@ linkis:
krb5: /etc/krb5.keytab
spark:
version: 2.4.3
+ configMapName: spark-conf
hive:
version: 2.3.3
+ configMapName: hive-conf
meta:
- url: "" # jdbc:mysql://localhost:3306/metastore?useUnicode=true
- user: "" # root
- password: "" # 123456
+ url: "jdbc:mysql://mysql.mysql.svc.cluster.local:3306/hive_metadata?&createDatabaseIfNotExist=true&characterEncoding=UTF-8&useSSL=false" # jdbc:mysql://localhost:3306/metastore?useUnicode=true
+ user: "root" # root
+ password: "123456" # 123456
+ zookeeper:
+ version: 3.5.9
+ configMapName: zookeeper-conf
+ flink:
+ version: 1.12.2
+ configMapName: flink-conf
mgEureka:
replicas: 1
@@ -276,51 +285,6 @@ psPublicService:
extras: []
froms: []
-psCs:
- replicas: 1
- port: 9108
- jvmHeapSize: "512M"
- prometheus:
- metricsPath: metrics
- annotations: {}
- nodeSelector: {}
- tolerations: []
- affinity: {}
- resources: {}
- envs:
- extras: []
- froms: []
-
-psDataSourceManager:
- replicas: 1
- port: 9109
- jvmHeapSize: "512M"
- prometheus:
- metricsPath: metrics
- annotations: {}
- nodeSelector: {}
- tolerations: []
- affinity: {}
- resources: {}
- envs:
- extras: []
- froms: []
-
-psMetadataQuery:
- replicas: 1
- port: 9110
- jvmHeapSize: "512M"
- prometheus:
- metricsPath: metrics
- annotations: {}
- nodeSelector: {}
- tolerations: []
- affinity: {}
- resources: {}
- envs:
- extras: []
- froms: []
-
Web:
replicas: 1
port: 8087
diff --git a/linkis-dist/helm/scripts/install-ldh.sh b/linkis-dist/helm/scripts/install-charts-with-ldh.sh
similarity index 63%
copy from linkis-dist/helm/scripts/install-ldh.sh
copy to linkis-dist/helm/scripts/install-charts-with-ldh.sh
index 9736b4c7a..1ca8a20aa 100755
--- a/linkis-dist/helm/scripts/install-ldh.sh
+++ b/linkis-dist/helm/scripts/install-charts-with-ldh.sh
@@ -19,20 +19,9 @@ WORK_DIR=`cd $(dirname $0); pwd -P`
. ${WORK_DIR}/common.sh
-set -e
+KUBE_NAMESPACE=${1:-linkis}
+HELM_RELEASE_NAME=${2:-linkis-demo}
-LDH_VERSION=${LDH_VERSION-${LINKIS_IMAGE_TAG}}
-echo "# LDH version: ${LINKIS_IMAGE_TAG}"
+sh ${WORK_DIR}/install-linkis.sh ${KUBE_NAMESPACE} ${HELM_RELEASE_NAME} false
-# load image
-if [ "X${KIND_LOAD_IMAGE}" == "Xtrue" ]; then
- echo "# Loading LDH image ..."
- kind load docker-image linkis-ldh:${LINKIS_IMAGE_TAG} --name ${KIND_CLUSTER_NAME}
-fi
-
-# deploy LDH
-echo "# Deploying LDH ..."
-kubectl create ns ldh
-kubectl apply -n ldh -f ${RESOURCE_DIR}/ldh/configmaps
-
-LDH_VERSION=${LDH_VERSION} envsubst < ${RESOURCE_DIR}/ldh/ldh.yaml | kubectl apply -n ldh -f -
+kubectl apply -n ${KUBE_NAMESPACE} -f ${RESOURCE_DIR}/ldh/configmaps
diff --git a/linkis-dist/helm/scripts/install-charts.sh b/linkis-dist/helm/scripts/install-charts.sh
index 9f25a433e..4f0642326 100755
--- a/linkis-dist/helm/scripts/install-charts.sh
+++ b/linkis-dist/helm/scripts/install-charts.sh
@@ -22,25 +22,4 @@ WORK_DIR=`cd $(dirname $0); pwd -P`
KUBE_NAMESPACE=${1:-linkis}
HELM_RELEASE_NAME=${2:-linkis-demo}
-. ${WORK_DIR}/common.sh
-
-if [ "X${HELM_DEBUG}" == "Xtrue" ]; then
- # template helm charts
- helm template --namespace ${KUBE_NAMESPACE} -f ${LINKIS_CHART_DIR}/values.yaml ${HELM_RELEASE_NAME} ${LINKIS_CHART_DIR}
-else
- # create hadoop configs
- if [ "X${WITH_LDH}" == "Xtrue" ]; then
- kubectl apply -n ${KUBE_NAMESPACE} -f ${RESOURCE_DIR}/ldh/configmaps
- fi
- # load image
- if [ "X${KIND_LOAD_IMAGE}" == "Xtrue" ]; then
- echo "# Loading Linkis image ..."
- kind load docker-image linkis:${LINKIS_IMAGE_TAG} --name ${KIND_CLUSTER_NAME}
- kind load docker-image linkis-web:${LINKIS_IMAGE_TAG} --name ${KIND_CLUSTER_NAME}
- fi
- # install helm charts
- helm install --create-namespace --namespace ${KUBE_NAMESPACE} \
- -f ${LINKIS_CHART_DIR}/values.yaml \
- --set image.tag=${LINKIS_IMAGE_TAG} \
- ${HELM_RELEASE_NAME} ${LINKIS_CHART_DIR}
-fi
+sh ${WORK_DIR}/install-linkis.sh ${KUBE_NAMESPACE} ${HELM_RELEASE_NAME} true
diff --git a/linkis-dist/helm/scripts/install-ldh.sh b/linkis-dist/helm/scripts/install-ldh.sh
index 9736b4c7a..39beab8a4 100755
--- a/linkis-dist/helm/scripts/install-ldh.sh
+++ b/linkis-dist/helm/scripts/install-ldh.sh
@@ -32,7 +32,12 @@ fi
# deploy LDH
echo "# Deploying LDH ..."
-kubectl create ns ldh
+set +e
+x=`kubectl get ns ldh 2> /dev/null`
+set -e
+if [ "X${x}" == "X" ]; then
+ kubectl create ns ldh
+fi
kubectl apply -n ldh -f ${RESOURCE_DIR}/ldh/configmaps
LDH_VERSION=${LDH_VERSION} envsubst < ${RESOURCE_DIR}/ldh/ldh.yaml | kubectl apply -n ldh -f -
diff --git a/linkis-dist/helm/scripts/install-charts.sh b/linkis-dist/helm/scripts/install-linkis.sh
similarity index 89%
copy from linkis-dist/helm/scripts/install-charts.sh
copy to linkis-dist/helm/scripts/install-linkis.sh
index 9f25a433e..75abd25ac 100755
--- a/linkis-dist/helm/scripts/install-charts.sh
+++ b/linkis-dist/helm/scripts/install-linkis.sh
@@ -21,6 +21,7 @@ WORK_DIR=`cd $(dirname $0); pwd -P`
KUBE_NAMESPACE=${1:-linkis}
HELM_RELEASE_NAME=${2:-linkis-demo}
+LOCAL_MODE=${3:-true}
. ${WORK_DIR}/common.sh
@@ -39,8 +40,9 @@ else
kind load docker-image linkis-web:${LINKIS_IMAGE_TAG} --name ${KIND_CLUSTER_NAME}
fi
# install helm charts
+ echo "# Installing linkis, image tag=${LINKIS_IMAGE_TAG},local mode=${LOCAL_MODE} ..."
helm install --create-namespace --namespace ${KUBE_NAMESPACE} \
-f ${LINKIS_CHART_DIR}/values.yaml \
- --set image.tag=${LINKIS_IMAGE_TAG} \
+ --set image.tag=${LINKIS_IMAGE_TAG},linkis.featureGates.localMode=${LOCAL_MODE} \
${HELM_RELEASE_NAME} ${LINKIS_CHART_DIR}
fi
diff --git a/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-hadoop.yaml b/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-hadoop.yaml
index 354e23ef3..aa5ff2e32 100644
--- a/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-hadoop.yaml
+++ b/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-hadoop.yaml
@@ -534,6 +534,12 @@ data:
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
+ <!--
+ <property>
+ <name>yarn.nodemanager.disk-health-checker.enable</name>
+ <value>false</value>
+ </property>
+ -->
</configuration>
capacity-scheduler.xml: |
<configuration>
diff --git a/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-spark.yaml b/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-spark.yaml
index 5d5185bfd..e1de98841 100644
--- a/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-spark.yaml
+++ b/linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-spark.yaml
@@ -172,37 +172,3 @@ data:
appender.console.filter.1.regex = .*Thrift error occurred during processing of message.*
appender.console.filter.1.onMatch = deny
appender.console.filter.1.onMismatch = neutral
-
- hive-site.xml: |
- <?xml version="1.0" encoding="UTF-8" standalone="no"?>
- <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
- <configuration>
- <property>
- <name>javax.jdo.option.ConnectionURL</name>
- <value>jdbc:mysql://mysql.mysql.svc.cluster.local:3306/hive_metadata?&createDatabaseIfNotExist=true&characterEncoding=UTF-8&useSSL=false</value>
- </property>
- <property>
- <name>javax.jdo.option.ConnectionUserName</name>
- <value>root</value>
- </property>
- <property>
- <name>javax.jdo.option.ConnectionPassword</name>
- <value>123456</value>
- </property>
- <property>
- <name>javax.jdo.option.ConnectionDriverName</name>
- <value>com.mysql.jdbc.Driver</value>
- </property>
- <property>
- <name>datanucleus.schema.autoCreateAll</name>
- <value>true</value>
- </property>
- <property>
- <name>hive.metastore.schema.verification</name>
- <value>false</value>
- </property>
- <property>
- <name>hive.metastore.uris</name>
- <value>thrift://ldh.ldh.svc.cluster.local:9083</value>
- </property>
- </configuration>
diff --git a/linkis-dist/helm/scripts/resources/ldh/ldh.yaml b/linkis-dist/helm/scripts/resources/ldh/ldh.yaml
index 6c2437e80..52eab4af4 100644
--- a/linkis-dist/helm/scripts/resources/ldh/ldh.yaml
+++ b/linkis-dist/helm/scripts/resources/ldh/ldh.yaml
@@ -82,77 +82,35 @@ spec:
- name: data-dir
emptyDir: {}
- name: hadoop-conf
- configMap:
- name: hadoop-conf
- items:
- - key: hadoop-env.sh
- path: hadoop-env.sh
- - key: core-site.xml
- path: core-site.xml
- - key: hdfs-site.xml
- path: hdfs-site.xml
- - key: yarn-env.sh
- path: yarn-env.sh
- - key: yarn-site.xml
- path: yarn-site.xml
- - key: capacity-scheduler.xml
- path: capacity-scheduler.xml
- - key: log4j.properties
- path: log4j.properties
+ projected:
+ sources:
+ - configMap:
+ name: hadoop-conf
- name: hive-conf
- configMap:
- name: hive-conf
- items:
- - key: hive-env.sh
- path: hive-env.sh
- - key: hive-site.xml
- path: hive-site.xml
- - key: hive-log4j2.properties
- path: hive-log4j2.properties
- - key: beeline-log4j2.properties
- path: beeline-log4j2.properties
- - key: hive-exec-log4j2.properties
- path: hive-exec-log4j2.properties
+ projected:
+ sources:
+ - configMap:
+ name: hive-conf
- name: spark-conf
- configMap:
- name: spark-conf
- items:
- - key: spark-env.sh
- path: spark-env.sh
- - key: hive-site.xml
- path: hive-site.xml
- - key: spark-defaults.conf
- path: spark-defaults.conf
- - key: log4j2.properties
- path: log4j2.properties
+ projected:
+ sources:
+ - configMap:
+ name: spark-conf
+ - configMap:
+ name: hive-conf
+ items:
+ - key: hive-site.xml
+ path: hive-site.xml
- name: flink-conf
- configMap:
- name: flink-conf
- items:
- - key: flink-conf.yaml
- path: flink-conf.yaml
- - key: log4j-cli.properties
- path: log4j-cli.properties
- - key: log4j-console.properties
- path: log4j-console.properties
- - key: log4j-session.properties
- path: log4j-session.properties
- - key: log4j.properties
- path: log4j.properties
- - key: logback-console.xml
- path: logback-console.xml
- - key: logback-session.xml
- path: logback-session.xml
- - key: logback.xml
- path: logback.xml
+ projected:
+ sources:
+ - configMap:
+ name: flink-conf
- name: zookeeper-conf
- configMap:
- name: zookeeper-conf
- items:
- - key: zoo.cfg
- path: zoo.cfg
- - key: log4j.properties
- path: log4j.properties
+ projected:
+ sources:
+ - configMap:
+ name: zookeeper-conf
containers:
- name: ldh
image: linkis-ldh:${LDH_VERSION}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org