You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by pe...@apache.org on 2022/06/24 08:20:26 UTC
[incubator-linkis] branch dev-1.2.0 updated: [LINKIS #1632] Add linkis-engineplugin-elasticsearch module (#2308)
This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.2.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.2.0 by this push:
new 0074b9ef9 [LINKIS #1632] Add linkis-engineplugin-elasticsearch module (#2308)
0074b9ef9 is described below
commit 0074b9ef9b4a4c36b38304dd9e9949fe689e2efb
Author: Zhen Wang <wa...@apache.org>
AuthorDate: Fri Jun 24 16:20:20 2022 +0800
[LINKIS #1632] Add linkis-engineplugin-elasticsearch module (#2308)
* [LINKIS #1632] Add linkis-engineplugin-elasticsearch module
* comments
---
.../manager/label/entity/engine/EngineType.scala | 2 +
.../manager/label/entity/engine/RunType.scala | 2 +
.../engineconn-plugins/elasticsearch/pom.xml | 162 ++++++++++++++++++++
.../src/main/assembly/distribution.xml | 73 +++++++++
.../main/resources/linkis-engineconn.properties | 23 +++
.../elasticsearch/src/main/resources/log4j2.xml | 91 ++++++++++++
.../ElasticSearchEngineConnPlugin.scala | 69 +++++++++
...asticSearchProcessEngineConnLaunchBuilder.scala | 37 ++---
.../conf/ElasticSearchConfiguration.scala | 48 ++++++
.../exception/EsConvertResponseException.scala | 33 +----
.../exception/EsEngineException.scala | 33 +----
.../exception/EsParamsIllegalException.scala | 33 +----
.../executer/ElasticSearchEngineConnExecutor.scala | 163 +++++++++++++++++++++
.../executer/ElasticSearchExecutorOrder.scala | 34 +----
.../executer/client/ElasticSearchExecutor.scala | 50 +++++++
.../executer/client/ElasticSearchResponse.scala | 38 ++---
.../elasticsearch/executer/client/EsClient.scala | 112 ++++++++++++++
.../executer/client/EsClientFactory.scala | 159 ++++++++++++++++++++
.../executer/client/ResponseHandler.scala | 77 ++++++++++
.../client/impl/ElasticSearchExecutorImpl.scala | 85 +++++++++++
.../executer/client/impl/ResponseHandlerImpl.scala | 150 +++++++++++++++++++
.../factory/ElasticSearchEngineConnFactory.scala | 38 +++++
.../factory/ElasticSearchJsonExecutorFactory.scala | 42 ++++++
.../factory/ElasticSearchSqlExecutorFactory.scala | 43 ++++++
24 files changed, 1439 insertions(+), 158 deletions(-)
diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala
index de1a84375..6f220a977 100644
--- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala
+++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala
@@ -57,6 +57,8 @@ object EngineType extends Enumeration with Logging {
val OPENLOOKENG = Value("openlookeng")
+ val ELASTICSEARCH = Value("elasticsearch")
+
def mapFsTypeToEngineType(fsType: String): String = {
fsType match {
case "file" =>
diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
index 3ff83ba38..e6af7bc70 100644
--- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
+++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
@@ -38,5 +38,7 @@ object RunType extends Enumeration {
val JAR = Value("jar")
val APPCONN = Value("appconn")
val FUNCTION_MDQ_TYPE = Value("function.mdq")
+ val ES_SQL = Value("essql")
+ val ES_JSON = Value("esjson")
}
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/pom.xml b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/pom.xml
new file mode 100644
index 000000000..db1b93ed1
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/pom.xml
@@ -0,0 +1,162 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>org.apache.linkis</groupId>
+ <version>1.1.2</version>
+ <relativePath>../../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>linkis-engineplugin-elasticsearch</artifactId>
+
+ <properties>
+ <elasticsearch.version>7.6.2</elasticsearch.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.linkis</groupId>
+ <artifactId>linkis-engineconn-plugin-core</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.linkis</groupId>
+ <artifactId>linkis-computation-engineconn</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.linkis</groupId>
+ <artifactId>linkis-storage</artifactId>
+ <version>${linkis.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.linkis</groupId>
+ <artifactId>linkis-rpc</artifactId>
+ <version>${linkis.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.linkis</groupId>
+ <artifactId>linkis-common</artifactId>
+ <version>${linkis.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- elasticsearch -->
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client-sniffer</artifactId>
+ <version>${elasticsearch.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>jackson-core</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-cbor</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-smile</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-csv</artifactId>
+ </dependency>
+
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.3</version>
+ <inherited>false</inherited>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
+ </executions>
+ <configuration>
+ <skipAssembly>false</skipAssembly>
+ <finalName>out</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ <includes>
+ <include>**/*.properties</include>
+ <include>**/*.xml</include>
+ <include>**/*.yml</include>
+ </includes>
+ </resource>
+ </resources>
+ </build>
+
+</project>
\ No newline at end of file
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/assembly/distribution.xml b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/assembly/distribution.xml
new file mode 100644
index 000000000..cf65b768a
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/assembly/distribution.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>linkis-engineplugin-elasticsearch</id>
+ <formats>
+ <format>dir</format>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>elasticsearch</baseDirectory>
+
+ <dependencySets>
+ <dependencySet>
+ <!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> -->
+ <!-- Now, select which projects to include in this module-set. -->
+ <outputDirectory>/dist/v${elasticsearch.version}/lib</outputDirectory>
+ <useProjectArtifact>true</useProjectArtifact>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <unpack>false</unpack>
+ <useStrictFiltering>false</useStrictFiltering>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+
+ <fileSet>
+ <directory>${basedir}/src/main/resources</directory>
+ <includes>
+ <include>linkis-engineconn.properties</include>
+ <include>log4j2.xml</include>
+ </includes>
+ <fileMode>0777</fileMode>
+ <outputDirectory>/dist/v${elasticsearch.version}/conf</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+
+ <fileSet>
+ <directory>${basedir}/target</directory>
+ <includes>
+ <include>*.jar</include>
+ </includes>
+ <excludes>
+ <exclude>*doc.jar</exclude>
+ </excludes>
+ <fileMode>0777</fileMode>
+ <outputDirectory>/plugin/${elasticsearch.version}</outputDirectory>
+ </fileSet>
+
+ </fileSets>
+
+</assembly>
+
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/resources/linkis-engineconn.properties b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/resources/linkis-engineconn.properties
new file mode 100644
index 000000000..81d279da2
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/resources/linkis-engineconn.properties
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+wds.linkis.server.version=v1
+#wds.linkis.engineconn.debug.enable=true
+wds.linkis.engineconn.plugin.default.class=org.apache.linkis.engineplugin.elasticsearch.ElasticSearchEngineConnPlugin
+wds.linkis.engineconn.support.parallelism=true
+
+# ElasticSearch
+linkis.es.cluster=127.0.0.1:9200
+
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/resources/log4j2.xml
new file mode 100644
index 000000000..020b94567
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/resources/log4j2.xml
@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration status="error" monitorInterval="30">
+ <appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
+ </Console>
+
+ <Send name="Send" >
+ <Filters>
+ <ThresholdFilter level="WARN" onMatch="ACCEPT" onMismatch="DENY" />
+ </Filters>
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
+ </Send>
+
+ <Send name="SendPackage" >
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
+ </Send>
+
+ <Console name="stderr" target="SYSTEM_ERR">
+ <ThresholdFilter level="ERROR" onMatch="ACCEPT" onMismatch="DENY" />
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} %-5level %class{36} %L %M - %msg%xEx%n"/>
+ </Console>
+ </appenders>
+
+ <loggers>
+ <root level="INFO">
+ <appender-ref ref="stderr"/>
+ <appender-ref ref="Console"/>
+ <appender-ref ref="Send"/>
+ </root>
+ <logger name="org.apache.hadoop.hive.ql.exec.StatsTask" level="info" additivity="true">
+ <appender-ref ref="SendPackage"/>
+ </logger>
+ <logger name="org.springframework.boot.diagnostics.LoggingFailureAnalysisReporter" level="error" additivity="true">
+ <appender-ref ref="stderr"/>
+ </logger>
+ <logger name="com.netflix.discovery" level="warn" additivity="true">
+ <appender-ref ref="Send"/>
+ </logger>
+ <logger name="org.apache.hadoop.yarn" level="warn" additivity="true">
+ <appender-ref ref="Send"/>
+ </logger>
+ <logger name="org.springframework" level="warn" additivity="true">
+ <appender-ref ref="Send"/>
+ </logger>
+ <logger name="org.apache.linkis.server.security" level="warn" additivity="true">
+ <appender-ref ref="Send"/>
+ </logger>
+ <logger name="org.apache.hadoop.hive.ql.exec.mr.ExecDriver" level="fatal" additivity="true">
+ <appender-ref ref="Send"/>
+ </logger>
+ <logger name="org.apache.hadoop.hdfs.KeyProviderCache" level="fatal" additivity="true">
+ <appender-ref ref="Send"/>
+ </logger>
+ <logger name="org.spark_project.jetty" level="ERROR" additivity="true">
+ <appender-ref ref="Send"/>
+ </logger>
+ <logger name="org.eclipse.jetty" level="ERROR" additivity="true">
+ <appender-ref ref="Send"/>
+ </logger>
+ <logger name="org.springframework" level="ERROR" additivity="true">
+ <appender-ref ref="Send"/>
+ </logger>
+ <logger name="org.reflections.Reflections" level="ERROR" additivity="true">
+ <appender-ref ref="Send"/>
+ </logger>
+
+ <logger name="org.apache.hadoop.ipc.Client" level="ERROR" additivity="true">
+ <appender-ref ref="Send"/>
+ </logger>
+
+ </loggers>
+</configuration>
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/ElasticSearchEngineConnPlugin.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/ElasticSearchEngineConnPlugin.scala
new file mode 100644
index 000000000..52208e010
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/ElasticSearchEngineConnPlugin.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.linkis.engineplugin.elasticsearch
+
+import java.util
+import org.apache.linkis.manager.engineplugin.common.EngineConnPlugin
+import org.apache.linkis.manager.engineplugin.common.creation.EngineConnFactory
+import org.apache.linkis.manager.engineplugin.common.launch.EngineConnLaunchBuilder
+import org.apache.linkis.manager.engineplugin.common.resource.{EngineResourceFactory, GenericEngineResourceFactory}
+import org.apache.linkis.manager.label.entity.Label
+import org.apache.linkis.manager.label.entity.engine.{EngineType, EngineTypeLabel}
+import org.apache.linkis.engineplugin.elasticsearch.builder.ElasticSearchProcessEngineConnLaunchBuilder
+import org.apache.linkis.engineplugin.elasticsearch.conf.ElasticSearchConfiguration
+import org.apache.linkis.engineplugin.elasticsearch.factory.ElasticSearchEngineConnFactory
+
+class ElasticSearchEngineConnPlugin extends EngineConnPlugin {
+
+ private var engineResourceFactory: EngineResourceFactory = _
+
+ private var engineFactory: EngineConnFactory = _
+
+ private val defaultLabels: util.List[Label[_]] = new util.ArrayList[Label[_]]()
+
+ private val resourceLocker = new Array[Byte](0)
+
+ private val engineFactoryLocker = new Array[Byte](0)
+
+ override def init(params: util.Map[String, Any]): Unit = {
+ val typeLabel = new EngineTypeLabel()
+ typeLabel.setEngineType(EngineType.ELASTICSEARCH.toString)
+ typeLabel.setVersion(ElasticSearchConfiguration.DEFAULT_VERSION.getValue)
+ this.defaultLabels.add(typeLabel)
+ }
+
+ override def getEngineResourceFactory: EngineResourceFactory = {
+ if (null == engineResourceFactory) resourceLocker.synchronized {
+ if (null == engineResourceFactory) engineResourceFactory = new GenericEngineResourceFactory
+ }
+ engineResourceFactory
+ }
+
+ override def getEngineConnLaunchBuilder: EngineConnLaunchBuilder = {
+ new ElasticSearchProcessEngineConnLaunchBuilder()
+ }
+
+ override def getEngineConnFactory: EngineConnFactory = {
+ if (null == engineFactory) engineFactoryLocker.synchronized {
+ if (null == engineFactory) engineFactory = new ElasticSearchEngineConnFactory
+ }
+ engineFactory
+ }
+
+ override def getDefaultLabels: util.List[Label[_]] = defaultLabels
+
+}
diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/builder/ElasticSearchProcessEngineConnLaunchBuilder.scala
similarity index 51%
copy from linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
copy to linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/builder/ElasticSearchProcessEngineConnLaunchBuilder.scala
index 3ff83ba38..76a0ac992 100644
--- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/builder/ElasticSearchProcessEngineConnLaunchBuilder.scala
@@ -5,38 +5,25 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.manager.label.entity.engine
+package org.apache.linkis.engineplugin.elasticsearch.builder
-object RunType extends Enumeration {
+import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder
+import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel
+import org.apache.linkis.storage.utils.StorageConfiguration
- type RunType = Value
- val SQL = Value("sql")
- val HIVE = Value("hql")
- val SCALA = Value("scala")
- val PYTHON = Value("python")
- val JAVA = Value("java")
- val PYSPARK = Value("py")
- val R = Value("r")
- val STORAGE = Value("out")
- val SHELL = Value("shell")
- val IO_FILE = Value("io_file")
- val IO_HDFS = Value("io_hdfs")
- val FPS = Value("fps")
- val PIPELINE = Value("pipeline")
- val JDBC = Value("jdbc")
- val PRESTO_SQL = Value("psql")
- val JAR = Value("jar")
- val APPCONN = Value("appconn")
- val FUNCTION_MDQ_TYPE = Value("function.mdq")
+class ElasticSearchProcessEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder {
+
+ override def getEngineStartUser(label: UserCreatorLabel): String = {
+ StorageConfiguration.HDFS_ROOT_USER.getValue
+ }
}
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/conf/ElasticSearchConfiguration.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/conf/ElasticSearchConfiguration.scala
new file mode 100644
index 000000000..7855d7bf7
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/conf/ElasticSearchConfiguration.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.linkis.engineplugin.elasticsearch.conf
+
+import org.apache.linkis.common.conf.{ByteType, CommonVars}
+
+object ElasticSearchConfiguration {
+
+ // es client
+ val ES_CLUSTER = CommonVars("linkis.es.cluster", "127.0.0.1:9200")
+ val ES_DATASOURCE_NAME = CommonVars("linkis.datasource", "default_datasource")
+ val ES_AUTH_CACHE = CommonVars("linkis.es.auth.cache", false)
+ val ES_USERNAME = CommonVars("linkis.es.username", "")
+ val ES_PASSWORD = CommonVars("linkis.es.password", "")
+ val ES_SNIFFER_ENABLE = CommonVars("linkis.es.sniffer.enable", false)
+ val ES_HTTP_METHOD = CommonVars("linkis.es.http.method", "GET")
+ val ES_HTTP_ENDPOINT = CommonVars("linkis.es.http.endpoint", "/_search")
+ val ES_HTTP_SQL_ENDPOINT = CommonVars("linkis.es.sql.endpoint", "/_sql")
+ val ES_SQL_FORMAT = CommonVars("linkis.es.sql.format", "{\"query\": \"%s\"}")
+ val ES_HTTP_HEADER_PREFIX = "linkis.es.headers."
+
+ // entrance resource
+ val ENTRANCE_MAX_JOB_INSTANCE = CommonVars("linkis.es.max.job.instance", 100)
+ val ENTRANCE_PROTECTED_JOB_INSTANCE = CommonVars("linkis.es.protected.job.instance", 20)
+ val ENGINE_DEFAULT_LIMIT = CommonVars("linkis.es.default.limit", 5000)
+
+ // resultSet
+ val ENGINE_RESULT_SET_MAX_CACHE = CommonVars("linkis.resultSet.cache.max", new ByteType("512k"))
+
+ val ENGINE_CONCURRENT_LIMIT = CommonVars[Int]("linkis.engineconn.concurrent.limit", 100)
+
+ val DEFAULT_VERSION = CommonVars[String]("linkis.engineconn.io.version", "7.6.2")
+
+}
diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsConvertResponseException.scala
similarity index 51%
copy from linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
copy to linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsConvertResponseException.scala
index 3ff83ba38..ab6b1a1c9 100644
--- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsConvertResponseException.scala
@@ -5,38 +5,17 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.manager.label.entity.engine
+package org.apache.linkis.engineplugin.elasticsearch.exception
-object RunType extends Enumeration {
+import org.apache.linkis.common.exception.ErrorException
- type RunType = Value
- val SQL = Value("sql")
- val HIVE = Value("hql")
- val SCALA = Value("scala")
- val PYTHON = Value("python")
- val JAVA = Value("java")
- val PYSPARK = Value("py")
- val R = Value("r")
- val STORAGE = Value("out")
- val SHELL = Value("shell")
- val IO_FILE = Value("io_file")
- val IO_HDFS = Value("io_hdfs")
- val FPS = Value("fps")
- val PIPELINE = Value("pipeline")
- val JDBC = Value("jdbc")
- val PRESTO_SQL = Value("psql")
- val JAR = Value("jar")
- val APPCONN = Value("appconn")
- val FUNCTION_MDQ_TYPE = Value("function.mdq")
-
-}
+case class EsConvertResponseException(errorMsg: String) extends ErrorException(70113, errorMsg)
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsEngineException.scala
similarity index 51%
copy from linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
copy to linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsEngineException.scala
index 3ff83ba38..a132ebcb8 100644
--- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsEngineException.scala
@@ -5,38 +5,17 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.manager.label.entity.engine
+package org.apache.linkis.engineplugin.elasticsearch.exception
-object RunType extends Enumeration {
+import org.apache.linkis.common.exception.ErrorException
- type RunType = Value
- val SQL = Value("sql")
- val HIVE = Value("hql")
- val SCALA = Value("scala")
- val PYTHON = Value("python")
- val JAVA = Value("java")
- val PYSPARK = Value("py")
- val R = Value("r")
- val STORAGE = Value("out")
- val SHELL = Value("shell")
- val IO_FILE = Value("io_file")
- val IO_HDFS = Value("io_hdfs")
- val FPS = Value("fps")
- val PIPELINE = Value("pipeline")
- val JDBC = Value("jdbc")
- val PRESTO_SQL = Value("psql")
- val JAR = Value("jar")
- val APPCONN = Value("appconn")
- val FUNCTION_MDQ_TYPE = Value("function.mdq")
-
-}
+case class EsEngineException(errorMsg: String) extends ErrorException(70114, errorMsg)
diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsParamsIllegalException.scala
similarity index 51%
copy from linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
copy to linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsParamsIllegalException.scala
index 3ff83ba38..27cef4027 100644
--- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsParamsIllegalException.scala
@@ -5,38 +5,17 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.manager.label.entity.engine
+package org.apache.linkis.engineplugin.elasticsearch.exception
-object RunType extends Enumeration {
+import org.apache.linkis.common.exception.ErrorException
- type RunType = Value
- val SQL = Value("sql")
- val HIVE = Value("hql")
- val SCALA = Value("scala")
- val PYTHON = Value("python")
- val JAVA = Value("java")
- val PYSPARK = Value("py")
- val R = Value("r")
- val STORAGE = Value("out")
- val SHELL = Value("shell")
- val IO_FILE = Value("io_file")
- val IO_HDFS = Value("io_hdfs")
- val FPS = Value("fps")
- val PIPELINE = Value("pipeline")
- val JDBC = Value("jdbc")
- val PRESTO_SQL = Value("psql")
- val JAR = Value("jar")
- val APPCONN = Value("appconn")
- val FUNCTION_MDQ_TYPE = Value("function.mdq")
-
-}
+case class EsParamsIllegalException(errorMsg: String) extends ErrorException(70112, errorMsg)
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/ElasticSearchEngineConnExecutor.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/ElasticSearchEngineConnExecutor.scala
new file mode 100644
index 000000000..bacbd2b83
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/ElasticSearchEngineConnExecutor.scala
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.linkis.engineplugin.elasticsearch.executer
+
+import java.util
+import java.util.concurrent.TimeUnit
+import com.google.common.cache.{Cache, CacheBuilder, RemovalListener, RemovalNotification}
+import org.apache.linkis.common.utils.{Logging, OverloadUtils, Utils}
+import org.apache.linkis.engineconn.common.conf.{EngineConnConf, EngineConnConstant}
+import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask
+import org.apache.linkis.engineconn.computation.executor.execute.{ConcurrentComputationExecutor, EngineExecutionContext}
+import org.apache.linkis.engineconn.core.EngineConnObject
+import org.apache.linkis.engineplugin.elasticsearch.executer.client.ElasticSearchErrorResponse
+import org.apache.linkis.governance.common.entity.ExecutionNodeStatus
+import org.apache.linkis.manager.common.entity.resource.{CommonNodeResource, LoadResource, NodeResource}
+import org.apache.linkis.manager.engineplugin.common.conf.EngineConnPluginConf
+import org.apache.linkis.manager.label.entity.Label
+import org.apache.linkis.protocol.engine.JobProgressInfo
+import org.apache.linkis.rpc.Sender
+import org.apache.linkis.scheduler.executer.{AliasOutputExecuteResponse, ErrorExecuteResponse, ExecuteResponse}
+import org.apache.linkis.storage.LineRecord
+import org.apache.linkis.storage.resultset.ResultSetFactory
+import org.apache.linkis.storage.resultset.table.TableMetaData
+import org.apache.commons.io.IOUtils
+import org.apache.linkis.engineplugin.elasticsearch.conf.ElasticSearchConfiguration
+import org.apache.linkis.engineplugin.elasticsearch.executer.client.{ElasticSearchErrorResponse, ElasticSearchExecutor, ElasticSearchJsonResponse, ElasticSearchTableResponse}
+import org.springframework.util.CollectionUtils
+
+import scala.collection.JavaConverters._
+
+class ElasticSearchEngineConnExecutor(override val outputPrintLimit: Int, val id: Int, runType: String) extends ConcurrentComputationExecutor(outputPrintLimit) with Logging {
+
+ private val executorLabels: util.List[Label[_]] = new util.ArrayList[Label[_]](2)
+
+ private val elasticSearchExecutorCache: Cache[String, ElasticSearchExecutor] = CacheBuilder.newBuilder()
+ .expireAfterAccess(EngineConnConf.ENGINE_TASK_EXPIRE_TIME.getValue, TimeUnit.MILLISECONDS)
+ .removalListener(new RemovalListener[String, ElasticSearchExecutor] {
+ override def onRemoval(notification: RemovalNotification[String, ElasticSearchExecutor]): Unit = {
+ notification.getValue.close
+ val task = getTaskById(notification.getKey)
+ if (!ExecutionNodeStatus.isCompleted(task.getStatus)) {
+ killTask(notification.getKey)
+ }
+ }
+ })
+ .maximumSize(EngineConnConstant.MAX_TASK_NUM).build()
+
+ override def init(): Unit = {
+ super.init()
+ }
+
+ override def execute(engineConnTask: EngineConnTask): ExecuteResponse = {
+ val elasticSearchExecutor = ElasticSearchExecutor(runType, engineConnTask.getProperties)
+ elasticSearchExecutor.open
+ elasticSearchExecutorCache.put(engineConnTask.getTaskId, elasticSearchExecutor)
+ super.execute(engineConnTask)
+ }
+
+ override def executeLine(engineExecutorContext: EngineExecutionContext, code: String): ExecuteResponse = {
+ val taskId = engineExecutorContext.getJobId.get
+ val elasticSearchExecutor = elasticSearchExecutorCache.getIfPresent(taskId)
+ val elasticSearchResponse = elasticSearchExecutor.executeLine(code)
+
+ elasticSearchResponse match {
+ case ElasticSearchTableResponse(columns, records) =>
+ val metaData = new TableMetaData(columns)
+ val resultSetWriter = engineExecutorContext.createResultSetWriter(ResultSetFactory.TABLE_TYPE)
+ resultSetWriter.addMetaData(metaData)
+ records.foreach(record => resultSetWriter.addRecord(record))
+ val output = resultSetWriter.toString
+ Utils.tryQuietly {
+ IOUtils.closeQuietly(resultSetWriter)
+ }
+ AliasOutputExecuteResponse(null, output)
+ case ElasticSearchJsonResponse(content) =>
+ val resultSetWriter = engineExecutorContext.createResultSetWriter(ResultSetFactory.TEXT_TYPE)
+ resultSetWriter.addMetaData(null)
+ content.split("\\n").foreach(item => resultSetWriter.addRecord(new LineRecord(item)))
+ val output = resultSetWriter.toString
+ Utils.tryQuietly {
+ IOUtils.closeQuietly(resultSetWriter)
+ }
+ AliasOutputExecuteResponse(null, output)
+ case ElasticSearchErrorResponse(message, body, cause) =>
+ ErrorExecuteResponse(message, cause)
+ }
+ }
+
+ override def executeCompletely(engineExecutorContext: EngineExecutionContext, code: String, completedLine: String): ExecuteResponse = null
+
+ override def progress(taskID: String): Float = 0.0f
+
+ override def getProgressInfo(taskID: String): Array[JobProgressInfo] = Array.empty[JobProgressInfo]
+
+ override def getExecutorLabels(): util.List[Label[_]] = executorLabels
+
+ override def setExecutorLabels(labels: util.List[Label[_]]): Unit = {
+ if (!CollectionUtils.isEmpty(labels)) {
+ executorLabels.clear()
+ executorLabels.addAll(labels)
+ }
+ }
+
+ override def supportCallBackLogs(): Boolean = false
+
+ override def requestExpectedResource(expectedResource: NodeResource): NodeResource = null
+
+ override def getCurrentNodeResource(): NodeResource = {
+ val properties = EngineConnObject.getEngineCreationContext.getOptions
+ if (properties.containsKey(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key)) {
+ val settingClientMemory = properties.get(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key)
+ if (!settingClientMemory.toLowerCase().endsWith("g")) {
+ properties.put(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key, settingClientMemory + "g")
+ }
+ }
+ val resource = new CommonNodeResource
+ val usedResource = new LoadResource(OverloadUtils.getProcessMaxMemory, 1)
+ resource.setUsedResource(usedResource)
+ resource
+ }
+
+ override def getId(): String = Sender.getThisServiceInstance.getInstance + s"_$id"
+
+ override def getConcurrentLimit: Int = ElasticSearchConfiguration.ENGINE_CONCURRENT_LIMIT.getValue
+
+ override def killTask(taskId: String): Unit = {
+ Utils.tryAndWarn {
+ val elasticSearchExecutor = elasticSearchExecutorCache.getIfPresent(taskId)
+ if (null != elasticSearchExecutor) {
+ elasticSearchExecutor.close
+ }
+ }
+ super.killTask(taskId)
+ }
+
+ override def killAll(): Unit = {
+ elasticSearchExecutorCache.asMap()
+ .values().asScala
+ .foreach(e => e.close)
+ }
+
+ override def transformTaskStatus(task: EngineConnTask, newStatus: ExecutionNodeStatus): Unit = {
+ super.transformTaskStatus(task, newStatus)
+ if (ExecutionNodeStatus.isCompleted(newStatus)) {
+ elasticSearchExecutorCache.invalidate(task.getTaskId)
+ }
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/ElasticSearchExecutorOrder.scala
similarity index 51%
copy from linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
copy to linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/ElasticSearchExecutorOrder.scala
index 3ff83ba38..e933681d2 100644
--- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/ElasticSearchExecutorOrder.scala
@@ -5,38 +5,20 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.manager.label.entity.engine
+package org.apache.linkis.engineplugin.elasticsearch.executer
-object RunType extends Enumeration {
-
- type RunType = Value
- val SQL = Value("sql")
- val HIVE = Value("hql")
- val SCALA = Value("scala")
- val PYTHON = Value("python")
- val JAVA = Value("java")
- val PYSPARK = Value("py")
- val R = Value("r")
- val STORAGE = Value("out")
- val SHELL = Value("shell")
- val IO_FILE = Value("io_file")
- val IO_HDFS = Value("io_hdfs")
- val FPS = Value("fps")
- val PIPELINE = Value("pipeline")
- val JDBC = Value("jdbc")
- val PRESTO_SQL = Value("psql")
- val JAR = Value("jar")
- val APPCONN = Value("appconn")
- val FUNCTION_MDQ_TYPE = Value("function.mdq")
+object ElasticSearchExecutorOrder extends Enumeration {
+ type ElasticSearchExecutorOrder = Value
+ val SQL = Value(1)
+ val JSON = Value(3)
}
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/ElasticSearchExecutor.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/ElasticSearchExecutor.scala
new file mode 100644
index 000000000..f1e874e06
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/ElasticSearchExecutor.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.linkis.engineplugin.elasticsearch.executer.client
+
+import java.io.IOException
+import java.util
+import org.apache.linkis.common.utils.Logging
+import org.apache.linkis.scheduler.executer.ExecuteResponse
+import org.apache.linkis.engineplugin.elasticsearch.executer.client.impl.ElasticSearchExecutorImpl
+
+import scala.collection.JavaConverters._
+
+trait ElasticSearchExecutor extends Logging {
+
+ @throws(classOf[IOException])
+ def open : Unit
+
+ def executeLine(code: String): ElasticSearchResponse
+
+ def close: Unit
+
+}
+
+object ElasticSearchExecutor {
+
+ def apply(runType: String, properties: util.Map[String, Object]): ElasticSearchExecutor = {
+ val newProperties = new util.HashMap[String, String]()
+ properties.asScala.foreach {
+ case (key: String, value: Object) if value != null =>
+ newProperties.put(key, String.valueOf(value))
+ case _ =>
+ }
+ new ElasticSearchExecutorImpl(runType, newProperties)
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/ElasticSearchResponse.scala
similarity index 51%
copy from linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
copy to linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/ElasticSearchResponse.scala
index 3ff83ba38..f8ad53b61 100644
--- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/ElasticSearchResponse.scala
@@ -5,38 +5,24 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.manager.label.entity.engine
+package org.apache.linkis.engineplugin.elasticsearch.executer.client
-object RunType extends Enumeration {
+import org.apache.linkis.storage.domain.Column
+import org.apache.linkis.storage.resultset.table.TableRecord
- type RunType = Value
- val SQL = Value("sql")
- val HIVE = Value("hql")
- val SCALA = Value("scala")
- val PYTHON = Value("python")
- val JAVA = Value("java")
- val PYSPARK = Value("py")
- val R = Value("r")
- val STORAGE = Value("out")
- val SHELL = Value("shell")
- val IO_FILE = Value("io_file")
- val IO_HDFS = Value("io_hdfs")
- val FPS = Value("fps")
- val PIPELINE = Value("pipeline")
- val JDBC = Value("jdbc")
- val PRESTO_SQL = Value("psql")
- val JAR = Value("jar")
- val APPCONN = Value("appconn")
- val FUNCTION_MDQ_TYPE = Value("function.mdq")
+trait ElasticSearchResponse
-}
+case class ElasticSearchTableResponse(columns: Array[Column], records: Array[TableRecord]) extends ElasticSearchResponse
+
+case class ElasticSearchJsonResponse(value: String) extends ElasticSearchResponse
+
+case class ElasticSearchErrorResponse(message: String, body: String = null, cause: Throwable = null) extends ElasticSearchResponse
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/EsClient.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/EsClient.scala
new file mode 100644
index 000000000..48cec78f6
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/EsClient.scala
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.linkis.engineplugin.elasticsearch.executer.client
+
+import java.nio.charset.StandardCharsets.UTF_8
+
+import org.apache.linkis.common.utils.Utils
+import org.apache.linkis.engineplugin.elasticsearch.conf.ElasticSearchConfiguration._
+import org.apache.linkis.server.JMap
+import org.apache.commons.codec.binary.Base64
+import org.apache.commons.lang3.StringUtils
+import org.apache.http.Header
+import org.apache.http.auth.{AUTH, Credentials, UsernamePasswordCredentials}
+import org.apache.http.message.BufferedHeader
+import org.apache.http.util.{CharArrayBuffer, EncodingUtils}
+import org.elasticsearch.client.{Cancellable, Request, RequestOptions, ResponseListener, RestClient}
+import org.elasticsearch.client.sniff.Sniffer
+
+import scala.collection.JavaConverters._
+
+trait EsClientOperate {
+
+ def execute(code: String, options: JMap[String, String], responseListener: ResponseListener): Cancellable
+
+ def close(): Unit
+
+}
+
+abstract class EsClient(datasourceName: String, client: RestClient, sniffer: Sniffer) extends EsClientOperate {
+
+ def getDatasourceName: String = datasourceName
+
+ def getRestClient: RestClient = client
+
+ def getSniffer: Sniffer = sniffer
+
+ override def close(): Unit = Utils.tryQuietly {
+ sniffer match {
+ case s: Sniffer => s.close()
+ case _ =>
+ }
+ client match {
+ case c: RestClient => c.close()
+ case _ =>
+ }
+ }
+
+}
+
+class EsClientImpl(datasourceName: String, client: RestClient, sniffer: Sniffer)
+ extends EsClient(datasourceName, client, sniffer) {
+
+ override def execute(code: String, options: JMap[String, String], responseListener: ResponseListener): Cancellable = {
+ val request = createRequest(code, options)
+ client.performRequestAsync(request, responseListener)
+ }
+
+ private def createRequest(code: String, options: JMap[String, String]): Request = {
+ val endpoint = ES_HTTP_ENDPOINT.getValue(options)
+ val method = ES_HTTP_METHOD.getValue(options)
+ val request = new Request(method, endpoint)
+ request.setOptions(getRequestOptions(options))
+ request.setJsonEntity(code)
+ request
+ }
+
+ private def getRequestOptions(options: JMap[String, String]): RequestOptions = {
+ val builder = RequestOptions.DEFAULT.toBuilder()
+
+ val username = ES_USERNAME.getValue(options)
+ val password = ES_PASSWORD.getValue(options)
+ // username / password convert to base auth
+ if (StringUtils.isNotBlank(username) && StringUtils.isNotBlank(password)) {
+ val authHeader = authenticate(new UsernamePasswordCredentials(username, password), UTF_8.name())
+ builder.addHeader(authHeader.getName, authHeader.getValue)
+ }
+
+ options.asScala.filter(entry => entry._1 != null && entry._2 != null && entry._1.startsWith(ES_HTTP_HEADER_PREFIX))
+ .foreach(entry => builder.addHeader(entry._1, entry._2))
+
+ builder.build()
+ }
+
+ private def authenticate(credentials: Credentials, charset: String): Header = {
+ val tmp = new StringBuilder
+ tmp.append(credentials.getUserPrincipal.getName)
+ tmp.append(":")
+ tmp.append(if (credentials.getPassword == null) "null"
+ else credentials.getPassword)
+ val base64password = Base64.encodeBase64(EncodingUtils.getBytes(tmp.toString, charset), false)
+ val buffer = new CharArrayBuffer(32)
+ buffer.append(AUTH.WWW_AUTH_RESP)
+ buffer.append(": Basic ")
+ buffer.append(base64password, 0, base64password.length)
+ new BufferedHeader(buffer)
+ }
+
+}
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/EsClientFactory.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/EsClientFactory.scala
new file mode 100644
index 000000000..d547cdebc
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/EsClientFactory.scala
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.linkis.engineplugin.elasticsearch.executer.client
+
+import java.util
+import java.util.Map
+import org.apache.linkis.common.conf.CommonVars
+import org.apache.linkis.engineplugin.elasticsearch.conf.ElasticSearchConfiguration._
+import org.apache.linkis.server.JMap
+import org.apache.commons.lang3.StringUtils
+import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials}
+import org.apache.http.client.CredentialsProvider
+import org.apache.http.impl.client.BasicCredentialsProvider
+import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
+import org.apache.http.message.BasicHeader
+import org.apache.http.{Header, HttpHost}
+import org.apache.linkis.engineplugin.elasticsearch.exception.EsParamsIllegalException
+import org.elasticsearch.client.sniff.Sniffer
+import org.elasticsearch.client.{RestClient, RestClientBuilder}
+
+import scala.collection.JavaConverters._
+
+object EsClientFactory {
+
+ def getRestClient(options: JMap[String, String]): EsClient = {
+ val key = getDatasourceName(options)
+ if (StringUtils.isBlank(key)) {
+ return defaultClient
+ }
+
+ if (!ES_CLIENT_MAP.containsKey(key)) {
+ ES_CLIENT_MAP synchronized {
+ if (!ES_CLIENT_MAP.containsKey(key)) {
+ cacheClient(createRestClient(options))
+ }
+ }
+ }
+
+ ES_CLIENT_MAP.get(key)
+ }
+
+ private val MAX_CACHE_CLIENT_SIZE = 20
+ private val ES_CLIENT_MAP: Map[String, EsClient] = new util.LinkedHashMap[String, EsClient]() {
+ override def removeEldestEntry(eldest: Map.Entry[String, EsClient]): Boolean = if (size > MAX_CACHE_CLIENT_SIZE) {
+ eldest.getValue.close()
+ true
+ } else {
+ false
+ }
+ }
+
+ private def getDatasourceName(options: JMap[String, String]): String = {
+ options.getOrDefault(ES_DATASOURCE_NAME.key, "")
+ }
+
+ private def cacheClient(client: EsClient) = {
+ ES_CLIENT_MAP.put(client.getDatasourceName, client)
+ }
+
+ private def createRestClient(options: JMap[String, String]): EsClient = {
+ val clusterStr = options.get(ES_CLUSTER.key)
+ if (StringUtils.isBlank(clusterStr)) {
+ throw EsParamsIllegalException("cluster is blank!")
+ }
+ val cluster = getCluster(clusterStr)
+ if (cluster.isEmpty) {
+ throw EsParamsIllegalException("cluster is empty!")
+ }
+ val username = options.get(ES_USERNAME.key)
+ val password = options.get(ES_PASSWORD.key)
+
+ if (ES_AUTH_CACHE.getValue) {
+ setAuthScope(cluster, username, password)
+ }
+
+ val httpHosts = cluster.map(item => new HttpHost(item._1, item._2))
+ val builder = RestClient.builder(httpHosts: _*)
+ .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
+ override def customizeHttpClient(httpAsyncClientBuilder: HttpAsyncClientBuilder): HttpAsyncClientBuilder = {
+ if (!ES_AUTH_CACHE.getValue) {
+ httpAsyncClientBuilder.disableAuthCaching
+ }
+// httpClientBuilder.setDefaultRequestConfig(RequestConfig.DEFAULT)
+// httpClientBuilder.setDefaultConnectionConfig(ConnectionConfig.DEFAULT)
+ httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider)
+ }
+ })
+ if (defaultHeaders != null) {
+ builder.setDefaultHeaders(defaultHeaders)
+ }
+ val client = builder.build
+
+ val sniffer = if (ES_SNIFFER_ENABLE.getValue(options)) {
+ Sniffer.builder(client).build
+ } else null
+
+ val datasourceName = getDatasourceName(options)
+ new EsClientImpl(datasourceName, client, sniffer)
+ }
+
+ private val credentialsProvider: CredentialsProvider = new BasicCredentialsProvider()
+
+ private val defaultClient = {
+ val cluster = ES_CLUSTER.getValue
+ if (StringUtils.isBlank(cluster)) {
+ null
+ } else {
+ val defaultOpts = new util.HashMap[String, String]()
+ defaultOpts.put(ES_CLUSTER.key, cluster)
+ defaultOpts.put(ES_DATASOURCE_NAME.key, ES_DATASOURCE_NAME.getValue)
+ defaultOpts.put(ES_USERNAME.key, ES_USERNAME.getValue)
+ defaultOpts.put(ES_PASSWORD.key, ES_PASSWORD.getValue)
+ val client = createRestClient(defaultOpts)
+ cacheClient(client)
+ client
+ }
+ }
+
+ private val defaultHeaders: Array[Header] = CommonVars.properties.entrySet().asScala
+ .filter(entry => entry.getKey != null && entry.getValue != null && entry.getKey.toString.startsWith(ES_HTTP_HEADER_PREFIX))
+ .map(entry => new BasicHeader(entry.getKey.toString, entry.getValue.toString)).toArray[Header]
+
+ // host1:port1,host2:port2 -> [(host1,port1),(host2,port2)]
+ private def getCluster(clusterStr: String): Array[(String, Int)] = if (StringUtils.isNotBlank(clusterStr)) {
+ clusterStr.split(",")
+ .map(value => {
+ val arr = value.split(":")
+ (arr(0), arr(1).toInt)
+ })
+ } else Array()
+
+ // set cluster auth
+ private def setAuthScope(cluster: Array[(String, Int)], username: String, password: String): Unit = if (cluster != null && !cluster.isEmpty
+ && StringUtils.isNotBlank(username)
+ && StringUtils.isNotBlank(password)) {
+ cluster.foreach{
+ case (host, port) => {
+ credentialsProvider.setCredentials(new AuthScope(host, port, AuthScope.ANY_REALM, AuthScope.ANY_SCHEME)
+ , new UsernamePasswordCredentials(username, password))
+ }
+ case _ =>
+ }
+ }
+
+}
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/ResponseHandler.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/ResponseHandler.scala
new file mode 100644
index 000000000..d150cd71d
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/ResponseHandler.scala
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.linkis.engineplugin.elasticsearch.executer.client
+
+import com.fasterxml.jackson.databind.node.JsonNodeType
+import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
+import com.fasterxml.jackson.dataformat.cbor.CBORFactory
+import com.fasterxml.jackson.dataformat.csv.CsvMapper
+import com.fasterxml.jackson.dataformat.smile.SmileFactory
+import com.fasterxml.jackson.dataformat.yaml.YAMLMapper
+import org.apache.linkis.common.utils.Logging
+import org.apache.linkis.storage.domain._
+import org.apache.linkis.engineplugin.elasticsearch.executer.client.impl.ResponseHandlerImpl
+import org.elasticsearch.client.Response
+
+trait ResponseHandler extends Logging {
+
+ def handle(response: Response): ElasticSearchResponse
+
+}
+
+
+object ResponseHandler {
+
+ val RESPONSE_HANDLER = new ResponseHandlerImpl()
+
+ def handle(response: Response): ElasticSearchResponse =
+ RESPONSE_HANDLER.handle(response)
+
+ val jsonMapper = new ObjectMapper()
+ val yamlMapper = new YAMLMapper()
+ val cborMapper = new ObjectMapper(new CBORFactory())
+ val smileMapper = new ObjectMapper(new SmileFactory())
+ val csvMapper = new CsvMapper()
+
+ def getNodeDataType(node: JsonNode): DataType = node.getNodeType match {
+ case JsonNodeType.ARRAY => ArrayType
+ case JsonNodeType.BINARY => BinaryType
+ case JsonNodeType.BOOLEAN => BooleanType
+ case JsonNodeType.NULL => NullType
+ case JsonNodeType.NUMBER => DecimalType
+ case JsonNodeType.OBJECT => StructType
+ case JsonNodeType.POJO => StructType
+ case JsonNodeType.STRING => StringType
+ case JsonNodeType.MISSING => StringType
+ case _ => StringType
+ }
+
+ def getNodeTypeByEsType(estype: String): DataType = estype.toLowerCase match {
+ case "long" | "integer" | "short" | "byte" | "double" | "float" | "half_float" | "scaled_float" => DecimalType
+ case "text" | "keyword" => StringType
+ case "date" => DateType
+ case "binary" => BinaryType
+ case _ => StringType
+ }
+
+ def getNodeValue(node: JsonNode): Any = node.getNodeType match {
+ case JsonNodeType.NUMBER => node.asDouble()
+ case JsonNodeType.NULL => null
+ case _ => node.toString().replaceAll("\n|\t", " ")
+ }
+
+}
\ No newline at end of file
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/impl/ElasticSearchExecutorImpl.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/impl/ElasticSearchExecutorImpl.scala
new file mode 100644
index 000000000..adf46bc46
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/impl/ElasticSearchExecutorImpl.scala
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.linkis.engineplugin.elasticsearch.executer.client.impl
+
+import java.util.concurrent.CountDownLatch
+import org.apache.linkis.common.utils.Utils
+import org.apache.linkis.engineplugin.elasticsearch.executer.client.ResponseHandler
+import org.apache.linkis.protocol.constants.TaskConstant
+import org.apache.linkis.scheduler.executer.{AliasOutputExecuteResponse, ErrorExecuteResponse, ExecuteResponse, SuccessExecuteResponse}
+import org.apache.linkis.server.JMap
+import org.apache.linkis.storage.utils.StorageUtils
+import org.apache.linkis.engineplugin.elasticsearch.conf.ElasticSearchConfiguration
+import org.apache.linkis.engineplugin.elasticsearch.exception.EsConvertResponseException
+import org.apache.linkis.engineplugin.elasticsearch.executer.client.{ElasticSearchErrorResponse, ElasticSearchExecutor, ElasticSearchResponse, EsClient, EsClientFactory, ResponseHandler}
+import org.elasticsearch.client.{Cancellable, Response, ResponseListener}
+
+class ElasticSearchExecutorImpl(runType: String, properties: JMap[String, String]) extends ElasticSearchExecutor {
+
+ private var client: EsClient = _
+ private var cancelable: Cancellable = _
+ private var user: String = _
+
+ override def open: Unit = {
+ this.client = EsClientFactory.getRestClient(properties)
+ this.user = properties.getOrDefault(TaskConstant.UMUSER, StorageUtils.getJvmUser)
+ runType.trim.toLowerCase match {
+ case "essql" | "sql" =>
+ properties.putIfAbsent(ElasticSearchConfiguration.ES_HTTP_ENDPOINT.key,
+ ElasticSearchConfiguration.ES_HTTP_SQL_ENDPOINT.getValue(properties))
+ case _ =>
+ }
+ }
+
+ override def executeLine(code: String): ElasticSearchResponse = {
+ val realCode = code.trim()
+ info(s"es client begins to run $runType code:\n ${realCode.trim}")
+ val countDown = new CountDownLatch(1)
+ var executeResponse: ElasticSearchResponse = ElasticSearchErrorResponse("INCOMPLETE")
+ cancelable = client.execute(realCode, properties, new ResponseListener {
+ override def onSuccess(response: Response): Unit = {
+ executeResponse = convertResponse(response)
+ countDown.countDown()
+ }
+ override def onFailure(exception: Exception): Unit = {
+ executeResponse = ElasticSearchErrorResponse("EsEngineExecutor execute fail. ", null, exception)
+ countDown.countDown()
+ }
+ })
+ countDown.await()
+ executeResponse
+ }
+
+ // convert response to executeResponse
+ private def convertResponse(response: Response): ElasticSearchResponse = Utils.tryCatch[ElasticSearchResponse]{
+ val statusCode = response.getStatusLine.getStatusCode
+ if (statusCode >= 200 && statusCode < 300) {
+ ResponseHandler.handle(response)
+ } else {
+ ElasticSearchErrorResponse("EsEngineExecutor convert response fail. response code: " + response.getStatusLine.getStatusCode)
+ }
+ } {
+ case t: Throwable =>
+ ElasticSearchErrorResponse("EsEngineExecutor convert response error.", null, t)
+ }
+
+ override def close: Unit = cancelable match {
+ case c: Cancellable => c.cancel()
+ case _ =>
+ }
+
+}
\ No newline at end of file
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/impl/ResponseHandlerImpl.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/impl/ResponseHandlerImpl.scala
new file mode 100644
index 000000000..4ebfb56bc
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executer/client/impl/ResponseHandlerImpl.scala
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.linkis.engineplugin.elasticsearch.executer.client.impl
+
+import java.nio.charset.Charset
+import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode}
+import org.apache.linkis.common.utils.Utils
+import org.apache.linkis.engineplugin.elasticsearch.executer.client.ResponseHandler._
+import org.apache.linkis.engineplugin.elasticsearch.executer.client.ResponseHandler
+import org.apache.linkis.storage.domain._
+import org.apache.linkis.storage.resultset.table.TableRecord
+import org.apache.http.entity.ContentType
+import org.apache.http.util.EntityUtils
+import org.apache.linkis.engineplugin.elasticsearch.exception.EsConvertResponseException
+import org.apache.linkis.engineplugin.elasticsearch.executer.client.{ElasticSearchJsonResponse, ElasticSearchResponse, ElasticSearchTableResponse, ResponseHandler}
+import org.elasticsearch.client.Response
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
+class ResponseHandlerImpl extends ResponseHandler {
+
+ // scalastyle:off
+ override def handle(response: Response): ElasticSearchResponse = {
+ val contentType = ContentType.get(response.getEntity).getMimeType.toLowerCase
+ val charSet = ContentType.get(response.getEntity).getCharset match {
+ case c: Charset => c
+ case _ => Charset.forName("UTF-8")
+ }
+ val contentBytes = EntityUtils.toByteArray(response.getEntity)
+
+ if (contentBytes == null || contentBytes.isEmpty) {
+ throw EsConvertResponseException("EsEngineExecutor convert response fail, response content is empty.")
+ }
+
+ val jsonNode = Utils.tryCatch{ contentType match {
+ case "application/yaml" =>
+ yamlMapper.readTree(contentBytes)
+ case "application/cbor" =>
+ cborMapper.readTree(contentBytes)
+ case "application/smile" =>
+ smileMapper.readTree(contentBytes)
+ case _ =>
+ jsonMapper.readTree(contentBytes)
+ }
+ } {
+ case t: Throwable => {
+ warn("deserialize response content error, {}", t)
+ null
+ }
+ }
+
+ if (jsonNode == null) {
+ ElasticSearchJsonResponse(new String(contentBytes, charSet))
+ }
+
+ var isTable = false
+ val columns = ArrayBuffer[Column]()
+ val records = new ArrayBuffer[TableRecord]
+
+ // es json runType response
+ jsonNode.at("/hits/hits") match {
+ case hits: ArrayNode => {
+ isTable = true
+ columns += Column("_index", StringType, "")
+ columns += Column("_type", StringType, "")
+ columns += Column("_id", StringType, "")
+ columns += Column("_score", DoubleType, "")
+ hits.asScala.foreach {
+ case obj: ObjectNode => {
+ val lineValues = new Array[Any](columns.length).toBuffer
+ obj.fields().asScala.foreach(entry => {
+ val key = entry.getKey
+ val value = entry.getValue
+ if ("_source".equals(key.trim)) {
+ value.fields().asScala.foreach(sourceEntry => {
+ val sourcekey = sourceEntry.getKey
+ val sourcevalue = sourceEntry.getValue
+ val index = columns.indexWhere(_.columnName.equals(sourcekey))
+ if (index < 0) {
+ columns += Column(sourcekey, getNodeDataType(sourcevalue), "")
+ lineValues += getNodeValue(sourcevalue)
+ } else {
+ lineValues(index) = getNodeValue(sourcevalue)
+ }
+ })
+ } else {
+ val index = columns.indexWhere(_.columnName.equals(key))
+ if (index < 0) {
+ columns += Column(key, getNodeDataType(value), "")
+ lineValues += getNodeValue(value)
+ } else {
+ lineValues(index) = getNodeValue(value)
+ }
+ }
+ })
+ records += new TableRecord(lineValues.toArray)
+ }
+ case _ =>
+ }
+ }
+ case _ =>
+ }
+
+ // es sql runType response
+ jsonNode.at("/rows") match {
+ case rows: ArrayNode => {
+ isTable = true
+ jsonNode.get("columns").asInstanceOf[ArrayNode]
+ .asScala
+ .foreach(node => {
+ val name = node.get("name").asText()
+ val estype = node.get("type").asText().trim
+ columns += Column(name, getNodeTypeByEsType(estype), "")
+ })
+ rows.asScala.foreach {
+ case row: ArrayNode => {
+ val lineValues = new ArrayBuffer[Any]()
+ row.asScala.foreach(node => lineValues += getNodeValue(node))
+ records += new TableRecord(lineValues.toArray)
+ }
+ case _ =>
+ }
+ }
+ case _ =>
+ }
+
+ // write result
+ if (isTable) {
+ ElasticSearchTableResponse(columns.toArray, records.toArray)
+ } else {
+ ElasticSearchJsonResponse(new String(contentBytes, charSet))
+ }
+ }
+ // scalastyle:on
+}
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchEngineConnFactory.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchEngineConnFactory.scala
new file mode 100644
index 000000000..704d4de1b
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchEngineConnFactory.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.linkis.engineplugin.elasticsearch.factory
+
+import org.apache.linkis.common.utils.Logging
+import org.apache.linkis.engineconn.common.creation.EngineCreationContext
+import org.apache.linkis.manager.engineplugin.common.creation.{ExecutorFactory, MultiExecutorEngineConnFactory}
+import org.apache.linkis.manager.label.entity.engine.EngineType
+import org.apache.linkis.manager.label.entity.engine.EngineType.EngineType
+
+class ElasticSearchEngineConnFactory extends MultiExecutorEngineConnFactory with Logging {
+ private val executorFactoryArray = Array[ExecutorFactory](new ElasticSearchJsonExecutorFactory, new ElasticSearchSqlExecutorFactory)
+
+ override def getExecutorFactories: Array[ExecutorFactory] = {
+ executorFactoryArray
+ }
+
+ override protected def getDefaultExecutorFactoryClass: Class[_ <: ExecutorFactory] =
+ classOf[ElasticSearchJsonExecutorFactory]
+
+ override protected def getEngineConnType: EngineType = EngineType.ELASTICSEARCH
+
+ override protected def createEngineConnSession(engineCreationContext: EngineCreationContext): Any = null
+}
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchJsonExecutorFactory.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchJsonExecutorFactory.scala
new file mode 100644
index 000000000..ce4d93054
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchJsonExecutorFactory.scala
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.linkis.engineplugin.elasticsearch.factory
+
+import org.apache.linkis.engineconn.common.creation.EngineCreationContext
+import org.apache.linkis.engineconn.common.engineconn.EngineConn
+import org.apache.linkis.engineconn.computation.executor.creation.ComputationExecutorFactory
+import org.apache.linkis.engineconn.computation.executor.execute.ComputationExecutor
+import org.apache.linkis.governance.common.paser.JsonCodeParser
+import org.apache.linkis.manager.label.entity.Label
+import org.apache.linkis.manager.label.entity.engine.RunType
+import org.apache.linkis.manager.label.entity.engine.RunType.RunType
+import org.apache.linkis.engineplugin.elasticsearch.conf.ElasticSearchConfiguration
+import org.apache.linkis.engineplugin.elasticsearch.executer.ElasticSearchEngineConnExecutor
+
+class ElasticSearchJsonExecutorFactory extends ComputationExecutorFactory {
+
+ override protected def newExecutor(id: Int, engineCreationContext: EngineCreationContext,
+ engineConn: EngineConn, labels: Array[Label[_]]): ComputationExecutor = {
+ val executor = new ElasticSearchEngineConnExecutor(ElasticSearchConfiguration.ENGINE_DEFAULT_LIMIT.getValue,
+ id, RunType.ES_JSON.toString)
+ executor.setCodeParser(new JsonCodeParser)
+ executor
+ }
+
+ override protected def getRunType: RunType = RunType.ES_JSON
+
+}
diff --git a/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchSqlExecutorFactory.scala b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchSqlExecutorFactory.scala
new file mode 100644
index 000000000..a6871e97e
--- /dev/null
+++ b/linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchSqlExecutorFactory.scala
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.linkis.engineplugin.elasticsearch.factory
+
+import org.apache.linkis.engineconn.common.creation.EngineCreationContext
+import org.apache.linkis.engineconn.common.engineconn.EngineConn
+import org.apache.linkis.engineconn.computation.executor.creation.ComputationExecutorFactory
+import org.apache.linkis.engineconn.computation.executor.execute.ComputationExecutor
+import org.apache.linkis.engineconn.core.executor.ExecutorManager
+import org.apache.linkis.governance.common.paser.SQLCodeParser
+import org.apache.linkis.manager.label.entity.Label
+import org.apache.linkis.manager.label.entity.engine.RunType
+import org.apache.linkis.manager.label.entity.engine.RunType.RunType
+import org.apache.linkis.engineplugin.elasticsearch.conf.ElasticSearchConfiguration
+import org.apache.linkis.engineplugin.elasticsearch.executer.ElasticSearchEngineConnExecutor
+
+class ElasticSearchSqlExecutorFactory extends ComputationExecutorFactory {
+
+ override protected def newExecutor(id: Int, engineCreationContext: EngineCreationContext,
+ engineConn: EngineConn, labels: Array[Label[_]]): ComputationExecutor = {
+ val executor = new ElasticSearchEngineConnExecutor(ElasticSearchConfiguration.ENGINE_DEFAULT_LIMIT.getValue,
+ id, RunType.ES_SQL.toString)
+ executor.setCodeParser(new SQLCodeParser)
+ executor
+ }
+
+ override protected def getRunType: RunType = RunType.ES_SQL
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org