You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ki...@apache.org on 2021/07/28 03:00:46 UTC
[dolphinscheduler] branch 1.3.7-prepare updated: Query return
number In SQL should be configurable. (#5896)
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch 1.3.7-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/1.3.7-prepare by this push:
new 79e7c4f Query return number In SQL should be configurable. (#5896)
79e7c4f is described below
commit 79e7c4f0518e35e8e0501d22ef4376f2f2f179c4
Author: zhuangchong <37...@users.noreply.github.com>
AuthorDate: Wed Jul 28 11:00:36 2021 +0800
Query return number In SQL should be configurable. (#5896)
---
.../common/task/sql/SqlParameters.java | 49 +++++++++++++-------
.../common/task/SqlParametersTest.java | 3 ++
.../server/worker/task/sql/SqlTask.java | 53 ++++++----------------
.../home/pages/dag/_source/formModel/tasks/sql.vue | 24 ++++++++++
.../src/js/module/i18n/locale/en_US.js | 3 ++
.../src/js/module/i18n/locale/zh_CN.js | 3 ++
6 files changed, 78 insertions(+), 57 deletions(-)
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java
index 37f46f9..c494496 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.common.task.sql;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
@@ -24,7 +25,7 @@ import java.util.ArrayList;
import java.util.List;
/**
- * Sql/Hql parameter
+ * Sql/Hql parameter.
*/
public class SqlParameters extends AbstractParameters {
/**
@@ -103,6 +104,19 @@ public class SqlParameters extends AbstractParameters {
*/
private String receiversCc;
+ /**
+ * query result limit
+ */
+ private int limit;
+
+ public int getLimit() {
+ return limit;
+ }
+
+ public void setLimit(int limit) {
+ this.limit = limit;
+ }
+
public String getType() {
return type;
}
@@ -226,21 +240,22 @@ public class SqlParameters extends AbstractParameters {
@Override
public String toString() {
- return "SqlParameters{" +
- "type='" + type + '\'' +
- ", datasource=" + datasource +
- ", sql='" + sql + '\'' +
- ", sqlType=" + sqlType +
- ", sendEmail=" + sendEmail +
- ", displayRows=" + displayRows +
- ", udfs='" + udfs + '\'' +
- ", showType='" + showType + '\'' +
- ", connParams='" + connParams + '\'' +
- ", title='" + title + '\'' +
- ", receivers='" + receivers + '\'' +
- ", receiversCc='" + receiversCc + '\'' +
- ", preStatements=" + preStatements +
- ", postStatements=" + postStatements +
- '}';
+ return "SqlParameters{"
+ + "type='" + type + '\''
+ + ", datasource=" + datasource
+ + ", sql='" + sql + '\''
+ + ", sqlType=" + sqlType
+ + ", sendEmail=" + sendEmail
+ + ", displayRows=" + displayRows
+ + ", limit=" + limit
+ + ", udfs='" + udfs + '\''
+ + ", showType='" + showType + '\''
+ + ", connParams='" + connParams + '\''
+ + ", title='" + title + '\''
+ + ", receivers='" + receivers + '\''
+ + ", receiversCc='" + receiversCc + '\''
+ + ", preStatements=" + preStatements
+ + ", postStatements=" + postStatements
+ + '}';
}
}
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java
index 3c1612c..546bb63 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java
@@ -32,6 +32,7 @@ public class SqlParametersTest {
private final int sqlType = 0;
private final Boolean sendEmail = true;
private final int displayRows = 10;
+ private final int limit = 0;
private final String showType = "TABLE";
private final String title = "sql test";
@@ -49,6 +50,7 @@ public class SqlParametersTest {
sqlParameters.setDisplayRows(displayRows);
sqlParameters.setShowType(showType);
sqlParameters.setTitle(title);
+ sqlParameters.setLimit(limit);
Assert.assertEquals(type, sqlParameters.getType());
Assert.assertEquals(sql, sqlParameters.getSql());
@@ -59,6 +61,7 @@ public class SqlParametersTest {
Assert.assertEquals(displayRows, sqlParameters.getDisplayRows());
Assert.assertEquals(showType, sqlParameters.getShowType());
Assert.assertEquals(title, sqlParameters.getTitle());
+ Assert.assertEquals(limit, sqlParameters.getLimit());
Assert.assertTrue(sqlParameters.checkParameters());
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
index d410560..2b9b907 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
@@ -16,14 +16,9 @@
*/
package org.apache.dolphinscheduler.server.worker.task.sql;
-import static org.apache.dolphinscheduler.common.Constants.COMMA;
-import static org.apache.dolphinscheduler.common.Constants.HIVE_CONF;
-import static org.apache.dolphinscheduler.common.Constants.PASSWORD;
-import static org.apache.dolphinscheduler.common.Constants.SEMICOLON;
-import static org.apache.dolphinscheduler.common.Constants.STATUS;
-import static org.apache.dolphinscheduler.common.Constants.USER;
-import static org.apache.dolphinscheduler.common.enums.DbType.HIVE;
-
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.alert.utils.MailUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
@@ -35,11 +30,7 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.sql.SqlBinds;
import org.apache.dolphinscheduler.common.task.sql.SqlParameters;
import org.apache.dolphinscheduler.common.task.sql.SqlType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
-import org.apache.dolphinscheduler.common.utils.CommonUtils;
-import org.apache.dolphinscheduler.common.utils.EnumUtils;
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.common.utils.ParameterUtils;
+import org.apache.dolphinscheduler.common.utils.*;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
@@ -50,30 +41,16 @@ import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.utils.UDFUtils;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.slf4j.Logger;
-import org.apache.commons.lang.StringUtils;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Properties;
+import java.sql.*;
+import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import org.slf4j.Logger;
-
-import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONObject;
+import static org.apache.dolphinscheduler.common.Constants.*;
+import static org.apache.dolphinscheduler.common.enums.DbType.HIVE;
/**
* sql task
*/
@@ -97,11 +74,6 @@ public class SqlTask extends AbstractTask {
*/
private TaskExecutionContext taskExecutionContext;
- /**
- * default query sql limit
- */
- private static final int LIMIT = 10000;
-
public SqlTask(TaskExecutionContext taskExecutionContext, Logger logger) {
super(taskExecutionContext, logger);
@@ -124,14 +96,15 @@ public class SqlTask extends AbstractTask {
Thread.currentThread().setName(threadLoggerInfoName);
logger.info("Full sql parameters: {}", sqlParameters);
- logger.info("sql type : {}, datasource : {}, sql : {} , localParams : {},udfs : {},showType : {},connParams : {}",
+ logger.info("sql type : {}, datasource : {}, sql : {} , localParams : {},udfs : {},showType : {},connParams : {}, query max result limit : {}",
sqlParameters.getType(),
sqlParameters.getDatasource(),
sqlParameters.getSql(),
sqlParameters.getLocalParams(),
sqlParameters.getUdfs(),
sqlParameters.getShowType(),
- sqlParameters.getConnParams());
+ sqlParameters.getConnParams(),
+ sqlParameters.getLimit());
try {
SQLTaskExecutionContext sqlTaskExecutionContext = taskExecutionContext.getSqlTaskExecutionContext();
// load class
@@ -282,7 +255,7 @@ public class SqlTask extends AbstractTask {
int rowCount = 0;
- while (rowCount < LIMIT && resultSet.next()) {
+ while (rowCount < sqlParameters.getLimit() && resultSet.next()) {
JSONObject mapOfColValues = new JSONObject(true);
for (int i = 1; i <= num; i++) {
mapOfColValues.put(md.getColumnLabel(i), resultSet.getObject(i));
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue
index bfecfe3..9a6cb57 100644
--- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue
@@ -46,6 +46,13 @@
</div>
</div>
</m-list-box>
+ <m-list-box v-show="sqlType === '0'">
+ <div slot="text"><strong class='requiredIcon'>*</strong>{{$t('Max Numbers Return')}}</div>
+ <div slot="content">
+ <x-input type="input" :disabled="isDetails" size="medium" v-model="limit" :placeholder="$t('Max Numbers Return placeholder')">
+ </x-input>
+ </div>
+ </m-list-box>
<template v-if="sqlType==0 && sendEmail">
<m-list-box>
<div slot="text">{{$t('Show Type')}}</div>
@@ -187,6 +194,8 @@
sendEmail: false,
// Display rows
displayRows: 10,
+ // Max returned rows
+ limit: 10000,
// Email title
title: '',
// Form/attachment
@@ -210,6 +219,13 @@
},
methods: {
/**
+ * limit should't be empty;limit should be a non-negative number str;
+ * limit should be a number smaller or equal than Integer.MAX_VALUE in java.
+ */
+ isLimitInvalid () {
+ return !this.limit || !/^(0|[1-9]\d*)$/.test(this.limit) || parseInt(this.limit, 10) > 2147483647
+ },
+ /**
* return sqlType
*/
_onSqlType (a) {
@@ -270,6 +286,10 @@
this.$message.warning(`${i18n.$t('Mail subject required')}`)
return false
}
+ if (this.sqlType === '0' && this.isLimitInvalid()) {
+ this.$message.warning(`${i18n.$t('Max Numbers Return required')}`)
+ return false
+ }
if (this.sqlType==0 && this.sendEmail && !this.receivers.length) {
this.$message.warning(`${i18n.$t('Recipient required')}`)
return false
@@ -313,6 +333,7 @@
sqlType: this.sqlType,
sendEmail: this.sendEmail,
displayRows: this.displayRows,
+ limit: this.limit,
title: this.title,
receivers: this.receivers.join(','),
receiversCc: this.receiversCc.join(','),
@@ -390,6 +411,7 @@
sqlType: this.sqlType,
sendEmail: this.sendEmail,
displayRows: this.displayRows,
+ limit: this.limit,
title: this.title,
receivers: this.receivers.join(','),
receiversCc: this.receiversCc.join(','),
@@ -452,6 +474,7 @@
this.sqlType = o.params.sqlType
this.sendEmail = o.params.sendEmail || false
this.displayRows = o.params.displayRows || 10
+ this.limit = o.params.limit || 10000
this.connParams = o.params.connParams || ''
this.localParams = o.params.localParams || []
if(o.params.showType == '') {
@@ -498,6 +521,7 @@
sqlType: this.sqlType,
sendEmail: this.sendEmail,
displayRows: this.displayRows,
+ limit: this.limit,
title: this.title,
receivers: this.receivers.join(','),
receiversCc: this.receiversCc.join(','),
diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
index ec3125b..4af6644 100755
--- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
+++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
@@ -129,6 +129,9 @@ export default {
'Send Email': 'Send Email',
'Log display': 'Log display',
'rows of result': 'rows of result',
+ 'Max Numbers Return': 'Number of records to return',
+ 'Max Numbers Return placeholder': 'Default is 10000, a large value may cause high pressure on the memory',
+ 'Max Numbers Return required': 'Number of records to return parameter must be a number in the range of 0 - 2147483647',
'Show Type': 'Show Type',
Title: 'Title',
'Please enter the title of email': 'Please enter the title of email',
diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
index eeb6955..90b1eca 100755
--- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
+++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
@@ -129,6 +129,9 @@ export default {
'Send Email': '发送邮件',
'Log display': '日志显示',
'rows of result': '行查询结果',
+ 'Max Numbers Return': '返回的记录行数',
+ 'Max Numbers Return placeholder': '默认值10000,如果值过大可能会对内存造成较大压力',
+ 'Max Numbers Return required': '返回的记录行数值必须是一个在0-2147483647范围内的整数',
'Show Type': '呈现方式',
Title: '主题',
'Please enter the title of email': '请输入邮件主题',