You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by ca...@apache.org on 2022/08/08 06:11:00 UTC
[incubator-linkis-website] branch dev updated: [ISSUES-469]add presto.md (#470)
This is an automated email from the ASF dual-hosted git repository.
casion pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-linkis-website.git
The following commit(s) were added to refs/heads/dev by this push:
new 218811e4fe [ISSUES-469]add presto.md (#470)
218811e4fe is described below
commit 218811e4fecb94acbaeb9512efb8e0d6ef472cb4
Author: 成彬彬 <10...@users.noreply.github.com>
AuthorDate: Mon Aug 8 14:10:56 2022 +0800
[ISSUES-469]add presto.md (#470)
* add presto.md
---
docs/engine_usage/presto.md | 162 +++++++++++++++++++++
.../current/engine_usage/presto.md | 135 ++++++++++++++---
static/Images-zh/EngineUsage/presto-console.png | Bin 0 -> 59566 bytes
static/Images-zh/EngineUsage/presto-file.png | Bin 0 -> 77025 bytes
static/Images-zh/EngineUsage/presto-psql.png | Bin 0 -> 39785 bytes
5 files changed, 280 insertions(+), 17 deletions(-)
diff --git a/docs/engine_usage/presto.md b/docs/engine_usage/presto.md
new file mode 100644
index 0000000000..9d5a38dd6c
--- /dev/null
+++ b/docs/engine_usage/presto.md
@@ -0,0 +1,162 @@
+---
+title: Presto Engine
+sidebar_position: 11
+---
+
+This article mainly introduces the configuration, deployment and use of the Presto engine in Linkis1.X.
+
+## 1. Environment preparation
+
+If you want to use the Presto engine on your server, you need to prepare the Presto connection information, such as the connection address, username and password of the Presto cluster, etc.
+
+## 2. Deployment and configuration
+
+### 2.1 version selection and compilation
+Note: Before compiling the Presto engine, a full build of the Linkis project is required
+This engine plug-in is not included in the released installation deployment package by default.
+You can follow this guide to deploy and install https://linkis.apache.org/zh-CN/blog/2022/04/15/how-to-download-engineconn-plugin
+ or follow the steps below to manually compile and deploy
+
+
+Compile the Presto engine separately
+
+````
+${linkis_code_dir}/linkis-enginepconn-lugins/engineconn-plugins/presto/
+mvn clean install
+````
+
+### 2.2 Deployment and loading of materials
+
+The engine package compiled in step 2.1 is located in
+```bash
+${linkis_code_dir}/linkis-engineconn-plugins/engineconn-plugins/jdbc/target/out/presto
+````
+Upload to the engine directory of the server
+```bash
+${LINKIS_HOME}/lib/linkis-engineplugins
+````
+And restart linkis-engineplugin (or refresh through the engine interface)
+```bash
+cd ${LINKIS_HOME}/sbin
+sh linkis-daemon.sh restart cg-engineplugin
+````
+Check whether the engine refresh is successful: You can check whether the last_update_time of the linkis_engine_conn_plugin_bml_resources table in the database is the time when the refresh is triggered.
+
+```sql
+#Login to the linkis database
+select * from linkis_cg_engine_conn_plugin_bml_resources
+````
+
+### 2.3 Engine tags
+
+Linkis1.X is done through tags, so we need to insert data into our database, and the insertion method is as follows.
+
+```bash
+SET @PRESTO_LABEL="presto-x";
+SET @PRESTO_ALL=CONCAT('*-*,',@PRESTO_LABEL);
+SET @PRESTO_IDE=CONCAT('*-IDE,',@PRESTO_LABEL);
+
+SET @PRESTO_NAME="presto";
+
+insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@PRESTO_ALL, 'OPTIONAL', 2, now(), now ());
+insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@PRESTO_IDE, 'OPTIONAL', 2, now(), now ());
+
+select @label_id := id from linkis_cg_manager_label where `label_value` = @PRESTO_IDE;
+insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 2);
+
+
+INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.presto.url','Presto cluster connection','presto connection address','http://127.0.0.1:8080','None',null,'presto',0,0, 1,'Data source configuration');
+INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.presto.catalog','Query's Catalog','presto-connected catalog','hive','None',null,'presto',0,0,1,'Data source configuration) ;
+INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.presto.schema','Query Schema','Database connection schema','','None',null,'presto',0,0,1,'Data source configuration);
+INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.presto.source','source used for query','database connection source','','None',null,'presto',0,0,1,'data source configuration') ;
+
+
+
+--engine -*
+insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`)
+(select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config
+INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type = @PRESTO_NAME and label_value = @PRESTO_ALL);
+
+--engine default configuration
+insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`)
+(select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation
+INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @PRESTO_ALL);
+````
+
+### 2.2 Presto engine related configuration
+
+| Configuration | Default | Description | Required |
+| -------------------------------------- | ---------- ----------- | -------------------------------------- ----- | -------- |
+| wds.linkis.presto.url | http://127.0.0.1:8080 | Presto cluster connection | true |
+| wds.linkis.presto.username | default | Presto cluster username | false |
+| wds.linkis.presto.password | none | Presto cluster password | false |
+| wds.linkis.presto.catalog | system | Catalog for queries | true |
+| wds.linkis.presto.schema | None | Query Schema | true |
+| wds.linkis.presto.source | global | source used by the query | false |
+| presto.session.query_max_total_memory | 8GB | query uses maximum memory | false |
+| wds.linkis.presto.http.connectTimeout | 60 | Presto client connect timeout (unit: seconds) | false |
+| wds.linkis.presto.http.readTimeout | 60 | Presto client read timeout (unit: seconds) | false |
+| wds.linkis.engineconn.concurrent.limit | 100 | The maximum number of concurrent Presto engines | false |
+## 3 The use of the engine
+
+### 3.1 Preparing for Operation
+
+If the default parameters are not satisfied, there are three ways to configure some basic parameters.
+
+#### 3.1.1 console configuration
+
+
+
+#### 3.1.2 Task interface configuration
+Submit the task interface and configure it through the parameter params.configuration.runtime
+
+```shell
+Example of http request parameters
+{
+ "executionContent": {"code": "show teblas;", "runType": "psql"},
+ "params": {
+ "variable": {},
+ "configuration": {
+ "runtime": {
+ "wds.linkis.presto.url":"http://127.0.0.1:9090",
+ "wds.linkis.presto.catalog ":"hive",
+ "wds.linkis.presto.schema ":"default",
+ "wds.linkis.presto.source ":""
+ }
+ }
+ },
+ "source": {"scriptPath": "file:///mnt/bdp/hadoop/1.sql"},
+ "labels": {
+ "engineType": "presto-0.234",
+ "userCreator": "hadoop-IDE"
+ }
+}
+````
+
+#### 3.1.3 File configuration
+Configure by modifying the linkis-engineconn.properties file in the directory install path/lib/linkis-engineconn-plugins/presto/dist/v0.234/conf/, as shown below:
+
+
+
+### 3.2 Task submission via Linkis-cli
+
+To submit a task through linkis-cli, you need to specify the corresponding EngineConn and CodeType tag types. The use of presto is as follows:
+
+- Note that the `engineType presto-0.234` engine version setting is prefixed to `presto-0.234` if the `presto` version is `0.234`
+
+```shell
+ sh ./bin/linkis-cli -engineType presto-0.234 -codeType psql -code 'show tables;' -submitUser hadoop -proxyUser hadoop
+````
+
+If the management console, task interface, and configuration file are not configured, they can be configured in the cli client through the `-runtimeMap` property
+```shell
+sh ./bin/linkis-cli -engineType presto-0.234 -codeType tsql -code 'show tables;' -runtimeMap wds.linkis.presto.url=http://172.22.32.11:50070 -runtimeMap wds.linkis.presto. catalog=hive -runtimeMap wds.linkis.presto.schema=default -runtimeMap wds.linkis.presto.catalog=hive -submitUser hadoop -proxyUser hadoop
+````
+
+For specific usage, please refer to: [Linkis CLI Manual](../user_guide/linkiscli-manual.md).
+
+### 3.3 Using via Scriptis
+
+Right-click the workspace module and select a new script of type `psql`
+
+
\ No newline at end of file
diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/current/engine_usage/presto.md b/i18n/zh-CN/docusaurus-plugin-content-docs/current/engine_usage/presto.md
index 0569343d59..5854a68e66 100644
--- a/i18n/zh-CN/docusaurus-plugin-content-docs/current/engine_usage/presto.md
+++ b/i18n/zh-CN/docusaurus-plugin-content-docs/current/engine_usage/presto.md
@@ -12,10 +12,9 @@ sidebar_position: 11
## 2. 部署和配置
### 2.1 版本的选择和编译
-注意: 编译 Presto 引擎之前需要进行 Linkis 项目全量编译
-发布的安装部署包中默认不包含此引擎插件，
-你可以按此指引部署安装 https://linkis.apache.org/zh-CN/blog/2022/04/15/how-to-download-engineconn-plugin
-，或者按以下流程，手动编译部署
+注意: 编译 Presto 引擎之前需要进行 Linkis 项目全量编译发布的安装部署包中默认不包含此引擎插件， 你可以按此指引部署安装
+https://linkis.apache.org/zh-CN/blog/2022/04/15/how-to-download-engineconn-plugin
+或者按以下流程，手动编译部署
单独编译 Presto 引擎
@@ -40,24 +39,126 @@ ${LINKIS_HOME}/lib/linkis-engineplugins
cd ${LINKIS_HOME}/sbin
sh linkis-daemon.sh restart cg-engineplugin
```
+
+检查引擎是否刷新成功：可以查看数据库中的linkis_engine_conn_plugin_bml_resources这张表的last_update_time是否为触发刷新的时间。
+
+```sql
+#登陆到linkis的数据库
+select * from linkis_cg_engine_conn_plugin_bml_resources
+```
+
### 2.3 引擎的标签
Linkis1.X是通过标签来进行的，所以需要在我们数据库中插入数据，插入的方式如下文所示。
-[EngineConnPlugin引擎插件安装](../deployment/engine-conn-plugin-installation)
+```bash
+SET @PRESTO_LABEL="presto-x";
+SET @PRESTO_ALL=CONCAT('*-*,',@PRESTO_LABEL);
+SET @PRESTO_IDE=CONCAT('*-IDE,',@PRESTO_LABEL);
+
+SET @PRESTO_NAME="presto";
+
+insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@PRESTO_ALL, 'OPTIONAL', 2, now(), now());
+insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@PRESTO_IDE, 'OPTIONAL', 2, now(), now());
+
+select @label_id := id from linkis_cg_manager_label where `label_value` = @PRESTO_IDE;
+insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 2);
+
+
+INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.presto.url','Presto 集群连接','presto连接地址','http://127.0.0.1:8080','None',null,'presto',0,0,1,'数据源配置');
+INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.presto.catalog','查询的 Catalog ','presto连接的catalog','hive','None',null,'presto',0,0,1,'数据源配置);
+INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.presto.schema',' 查询的 Schema ','数据库连接schema','','None',null,'presto',0,0,1,'数据源配置);
+INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.presto.source','查询使用的 source ','数据库连接source','','None',null,'presto',0,0,1,'数据源配置');
+
+
+
+-- engine -*
+insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`)
+(select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config
+INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type = @PRESTO_NAME and label_value = @PRESTO_ALL);
+
+-- engine default configuration
+insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`)
+(select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation
+INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @PRESTO_ALL);
+```
### 2.2 Presto 引擎相关配置
-| 配置 | 默认值 | 说明 |
-| -------------------------------------- | --------------------- | ------------------------------------------- |
-| wds.linkis.presto.url | http://127.0.0.1:8080 | Presto 集群连接 |
-| wds.linkis.presto.username | default | Presto 集群用户名 |
-| wds.linkis.presto.password | 无 | Presto 集群密码 |
-| wds.linkis.presto.catalog | system | 查询的 Catalog |
-| wds.linkis.presto.schema | 无 | 查询的 Schema |
-| wds.linkis.presto.source | global | 查询使用的 source |
-| presto.session.query_max_total_memory | 8GB | 查询使用最大的内存 |
-| wds.linkis.presto.http.connectTimeout | 60 | Presto 客户端的 connect timeout（单位：秒） |
-| wds.linkis.presto.http.readTimeout | 60 | Presto 客户端的 read timeout（单位：秒） |
-| wds.linkis.engineconn.concurrent.limit | 100 | Presto 引擎最大并发数 |
+| 配置 | 默认值 | 说明 | 是否必须 |
+| -------------------------------------- | --------------------- | ------------------------------------------- | -------- |
+| wds.linkis.presto.url | http://127.0.0.1:8080 | Presto 集群连接 | true |
+| wds.linkis.presto.username | default | Presto 集群用户名 | false |
+| wds.linkis.presto.password | 无 | Presto 集群密码 | false |
+| wds.linkis.presto.catalog | system | 查询的 Catalog | true |
+| wds.linkis.presto.schema | 无 | 查询的 Schema | true |
+| wds.linkis.presto.source | global | 查询使用的 source | false |
+| presto.session.query_max_total_memory | 8GB | 查询使用最大的内存 | false |
+| wds.linkis.presto.http.connectTimeout | 60 | Presto 客户端的 connect timeout（单位：秒） | false |
+| wds.linkis.presto.http.readTimeout | 60 | Presto 客户端的 read timeout（单位：秒） | false |
+| wds.linkis.engineconn.concurrent.limit | 100 | Presto 引擎最大并发数 | false |
+
+## 3 引擎的使用
+
+### 3.1 准备操作
+
+如果默认参数不满足时，有三种方式可以进行一些基础参数配置。
+
+#### 3.1.1 管理台配置
+
+
+
+#### 3.1.2 任务接口配置
+提交任务接口，通过参数params.configuration.runtime进行配置
+
+```shell
+http 请求参数示例
+{
+ "executionContent": {"code": "show teblas;", "runType": "psql"},
+ "params": {
+ "variable": {},
+ "configuration": {
+ "runtime": {
+ "wds.linkis.presto.url":"http://127.0.0.1:9090",
+ "wds.linkis.presto.catalog ":"hive",
+ "wds.linkis.presto.schema ":"default",
+ "wds.linkis.presto.source ":""
+ }
+ }
+ },
+ "source": {"scriptPath": "file:///mnt/bdp/hadoop/1.sql"},
+ "labels": {
+ "engineType": "presto-0.234",
+ "userCreator": "hadoop-IDE"
+ }
+}
+```
+
+#### 3.1.3 文件配置
+通过修改目录 install path/lib/linkis-engineconn-plugins/presto/dist/v0.234/conf/ 中的linkis-engineconn.properties 文件进行配置，如下图：
+
+
+
+### 3.2 通过Linkis-cli进行任务提交
+
+通过linkis-cli的方式提交任务，需要指定对应的EngineConn和CodeType标签类型，presto的使用如下：
+
+- 注意 `engineType presto-0.234` 引擎版本设置是有前缀的 如 `presto` 版本为`0.234` 则设置为 ` presto-0.234`
+
+```shell
+ sh ./bin/linkis-cli -engineType presto-0.234 -codeType psql -code 'show tables;' -submitUser hadoop -proxyUser hadoop
+```
+
+如果管理台，任务接口，配置文件，均未配置时可在 cli 客户端中通过`-runtimeMap`属性进行配置
+
+```shell
+sh ./bin/linkis-cli -engineType presto-0.234 -codeType tsql -code 'show tables;' -runtimeMap wds.linkis.presto.url=http://172.22.32.11:50070 -runtimeMap wds.linkis.presto.catalog=hive -runtimeMap wds.linkis.presto.schema=default -runtimeMap wds.linkis.presto.catalog=hive -submitUser hadoop -proxyUser hadoop
+```
+
+具体使用可以参考： [Linkis CLI Manual](../user_guide/linkiscli-manual.md).
+
+### 3.3 通过 Scriptis 使用
+
+工作空间模块右键选择新建一个类型为`psql`的脚本
+
\ No newline at end of file
diff --git a/static/Images-zh/EngineUsage/presto-console.png b/static/Images-zh/EngineUsage/presto-console.png
new file mode 100644
index 0000000000..d3a92de1a5
Binary files /dev/null and b/static/Images-zh/EngineUsage/presto-console.png differ
diff --git a/static/Images-zh/EngineUsage/presto-file.png b/static/Images-zh/EngineUsage/presto-file.png
new file mode 100644
index 0000000000..49c00b9965
Binary files /dev/null and b/static/Images-zh/EngineUsage/presto-file.png differ
diff --git a/static/Images-zh/EngineUsage/presto-psql.png b/static/Images-zh/EngineUsage/presto-psql.png
new file mode 100644
index 0000000000..505f0a7a8c
Binary files /dev/null and b/static/Images-zh/EngineUsage/presto-psql.png differ
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org