You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zh...@apache.org on 2022/08/03 11:57:35 UTC
[dolphinscheduler] 15/16: [Fix]Recovery Traffic Control Config (#10505)
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 7104063e0d055d4368ff13006099570fb4c39860
Author: 旺阳 <qi...@cisco.com>
AuthorDate: Mon Jun 20 18:43:48 2022 +0800
[Fix]Recovery Traffic Control Config (#10505)
* recovery traffic config
* run ci
* update
(cherry picked from commit 6e4b2e69927af70bccda5a2e70fe93d7b1a852e7)
---
docs/docs/en/architecture/configuration.md | 8 ++-
docs/docs/zh/architecture/configuration.md | 6 ++-
.../api/configuration/AppConfiguration.java | 2 +-
.../api/configuration/TrafficConfiguration.java | 62 ++++------------------
.../api/interceptor/RateLimitInterceptor.java | 6 +--
.../src/main/resources/application.yaml | 15 ++++++
.../configuration/TrafficConfigurationTest.java | 6 +--
.../api/interceptor/RateLimitInterceptorTest.java | 8 +--
.../src/main/resources/application.yaml | 14 +++++
9 files changed, 63 insertions(+), 64 deletions(-)
diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md
index 85a45d0ab4..b41578438f 100644
--- a/docs/docs/en/architecture/configuration.md
+++ b/docs/docs/en/architecture/configuration.md
@@ -223,8 +223,14 @@ security.authentication.ldap.urls|ldap://ldap.forumsys.com:389/|LDAP urls
security.authentication.ldap.base.dn|dc=example,dc=com|LDAP base dn
security.authentication.ldap.username|cn=read-only-admin,dc=example,dc=com|LDAP username
security.authentication.ldap.password|password|LDAP password
-security.authentication.ldap.user.identity.attribute|uid|LDAP user identity attribute
+security.authentication.ldap.user.identity.attribute|uid|LDAP user identity attribute
security.authentication.ldap.user.email.attribute|mail|LDAP user email attribute
+security.authentication.ldap.user.not-exist-action|CREATE|action when LDAP user is not exist. Default CREATE: automatically create user when user not exist, DENY: deny log-in when user not exist
+traffic.control.global.switch|false|traffic control global switch
+traffic.control.max-global-qps-rate|300|global max request number per second
+traffic.control.tenant-switch|false|traffic control tenant switch
+traffic.control.default-tenant-qps-rate|10|default tenant max request number per second
+traffic.control.customize-tenant-qps-rate||customize tenant max request number per second
### master.properties [master-service log config]
diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md
index 7bcf697f34..cebb8c00ef 100644
--- a/docs/docs/zh/architecture/configuration.md
+++ b/docs/docs/zh/architecture/configuration.md
@@ -216,7 +216,11 @@ security.authentication.ldap.username|cn=read-only-admin,dc=example,dc=com|LDAP
security.authentication.ldap.password|password|LDAP密码
security.authentication.ldap.user.identity.attribute|uid|LDAP用户身份标识字段名
security.authentication.ldap.user.email.attribute|mail|LDAP邮箱字段名
-
+traffic.control.global.switch|false|流量控制全局开关
+traffic.control.max-global-qps-rate|300|全局最大请求数/秒
+traffic.control.tenant-switch|false|流量控制租户开关
+traffic.control.default-tenant-qps-rate|10|默认租户最大请求数/秒限制
+traffic.control.customize-tenant-qps-rate||自定义租户最大请求数/秒限制
## 6.master.properties [Master服务配置]
|参数 |默认值| 描述|
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AppConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AppConfiguration.java
index 105e3738df..2a80ef86ef 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AppConfiguration.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AppConfiguration.java
@@ -95,7 +95,7 @@ public class AppConfiguration implements WebMvcConfigurer {
public void addInterceptors(InterceptorRegistry registry) {
// i18n
registry.addInterceptor(localeChangeInterceptor());
- if (trafficConfiguration.isTrafficGlobalControlSwitch() || trafficConfiguration.isTrafficTenantControlSwitch()) {
+ if (trafficConfiguration.isGlobalSwitch() || trafficConfiguration.isTenantSwitch()) {
registry.addInterceptor(createRateLimitInterceptor());
}
registry.addInterceptor(loginInterceptor())
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java
index 75a9c25410..3b66a2d6ca 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java
@@ -17,62 +17,22 @@
package org.apache.dolphinscheduler.api.configuration;
+import lombok.Data;
+
+import java.util.HashMap;
import java.util.Map;
-import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
+@Data
@Configuration
+@ConfigurationProperties(prefix = "traffic.control")
public class TrafficConfiguration {
- @Value("${traffic.control.global.switch:false}")
- private boolean trafficGlobalControlSwitch;
- @Value("${traffic.control.max.global.qps.rate:300}")
- private Integer maxGlobalQpsRate;
- @Value("${traffic.control.tenant.switch:false}")
- private boolean trafficTenantControlSwitch;
- @Value("${traffic.control.default.tenant.qps.rate:10}")
- private Integer defaultTenantQpsRate;
- @Value("#{'${traffic.control.customize.tenant.qps.rate:}'.empty?null:'${traffic.control.customize.tenant.qps.rate:}'}")
- private Map<String, Integer> customizeTenantQpsRate;
-
- public boolean isTrafficGlobalControlSwitch() {
- return trafficGlobalControlSwitch;
- }
-
- public void setTrafficGlobalControlSwitch(boolean trafficGlobalControlSwitch) {
- this.trafficGlobalControlSwitch = trafficGlobalControlSwitch;
- }
-
- public Integer getMaxGlobalQpsRate() {
- return maxGlobalQpsRate;
- }
-
- public void setMaxGlobalQpsRate(Integer maxGlobalQpsRate) {
- this.maxGlobalQpsRate = maxGlobalQpsRate;
- }
-
- public boolean isTrafficTenantControlSwitch() {
- return trafficTenantControlSwitch;
- }
-
- public void setTrafficTenantControlSwitch(boolean trafficTenantControlSwitch) {
- this.trafficTenantControlSwitch = trafficTenantControlSwitch;
- }
-
- public Integer getDefaultTenantQpsRate() {
- return defaultTenantQpsRate;
- }
-
- public void setDefaultTenantQpsRate(Integer defaultTenantQpsRate) {
- this.defaultTenantQpsRate = defaultTenantQpsRate;
- }
-
- public Map<String, Integer> getCustomizeTenantQpsRate() {
- return customizeTenantQpsRate;
- }
-
- public void setCustomizeTenantQpsRate(Map<String, Integer> customizeTenantQpsRate) {
- this.customizeTenantQpsRate = customizeTenantQpsRate;
- }
+ private boolean globalSwitch;
+ private Integer maxGlobalQpsRate = 300;
+ private boolean tenantSwitch;
+ private Integer defaultTenantQpsRate = 10;
+ private Map<String, Integer> customizeTenantQpsRate = new HashMap<>();
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java
index 925412576c..9f6ee6fb43 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java
@@ -72,7 +72,7 @@ public class RateLimitInterceptor implements HandlerInterceptor {
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws ExecutionException {
// tenant-level rate limit
- if (trafficConfiguration.isTrafficTenantControlSwitch()) {
+ if (trafficConfiguration.isTenantSwitch()) {
String token = request.getHeader("token");
if (!StringUtils.isEmpty(token)) {
RateLimiter tenantRateLimiter = tenantRateLimiterCache.get(token);
@@ -84,7 +84,7 @@ public class RateLimitInterceptor implements HandlerInterceptor {
}
}
// global rate limit
- if (trafficConfiguration.isTrafficGlobalControlSwitch()) {
+ if (trafficConfiguration.isGlobalSwitch()) {
if (!globalRateLimiter.tryAcquire()) {
response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value());
logger.warn("Too many request, reach global rate limit, current qps is {}", globalRateLimiter.getRate());
@@ -96,7 +96,7 @@ public class RateLimitInterceptor implements HandlerInterceptor {
public RateLimitInterceptor(TrafficConfiguration trafficConfiguration) {
this.trafficConfiguration = trafficConfiguration;
- if (trafficConfiguration.isTrafficGlobalControlSwitch()) {
+ if (trafficConfiguration.isGlobalSwitch()) {
this.globalRateLimiter = RateLimiter.create(trafficConfiguration.getMaxGlobalQpsRate(), 1, TimeUnit.SECONDS);
}
}
diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml
index 3fff4539dd..a7f8e306cb 100644
--- a/dolphinscheduler-api/src/main/resources/application.yaml
+++ b/dolphinscheduler-api/src/main/resources/application.yaml
@@ -153,6 +153,21 @@ security:
user.identity.attribute: uid
user.email.attribute: mail
+# Traffic control, if you turn on this config, the maximum number of request/s will be limited.
+# global max request number per second
+# default tenant-level max request number
+traffic:
+ control:
+ global-switch: false
+ max-global-qps-rate: 300
+ tenant-switch: false
+ default-tenant-qps-rate: 10
+ #customize-tenant-qps-rate:
+ # eg.
+ #tenant1: 11
+ #tenant2: 20
+
+
# Override by profile
---
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/TrafficConfigurationTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/TrafficConfigurationTest.java
index bb0f6b79c5..a0c3d8a5a7 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/TrafficConfigurationTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/TrafficConfigurationTest.java
@@ -32,7 +32,7 @@ public class TrafficConfigurationTest extends AbstractControllerTest {
@Test
public void isTrafficGlobalControlSwitch() {
- Assert.assertFalse(trafficConfiguration.isTrafficGlobalControlSwitch());
+ Assert.assertFalse(trafficConfiguration.isGlobalSwitch());
}
@Test
@@ -42,7 +42,7 @@ public class TrafficConfigurationTest extends AbstractControllerTest {
@Test
public void isTrafficTenantControlSwitch() {
- Assert.assertFalse(trafficConfiguration.isTrafficTenantControlSwitch());
+ Assert.assertFalse(trafficConfiguration.isTenantSwitch());
}
@Test
@@ -54,4 +54,4 @@ public class TrafficConfigurationTest extends AbstractControllerTest {
public void getCustomizeTenantQpsRate() {
Assert.assertTrue(MapUtils.isEmpty(trafficConfiguration.getCustomizeTenantQpsRate()));
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptorTest.java
index 95ae3e3779..c903df5009 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptorTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptorTest.java
@@ -48,7 +48,7 @@ public class RateLimitInterceptorTest {
@Test
public void testPreHandleWithTenantLevenControl() throws ExecutionException {
TrafficConfiguration trafficConfiguration = new TrafficConfiguration();
- trafficConfiguration.setTrafficTenantControlSwitch(true);
+ trafficConfiguration.setTenantSwitch(true);
Map<String, Integer> map = new HashMap<>();
map.put("tenant1", 2);
map.put("tenant2", 2);
@@ -72,8 +72,8 @@ public class RateLimitInterceptorTest {
@Test
public void testPreHandleWithGlobalControl() throws ExecutionException {
TrafficConfiguration trafficConfiguration = new TrafficConfiguration();
- trafficConfiguration.setTrafficTenantControlSwitch(true);
- trafficConfiguration.setTrafficGlobalControlSwitch(true);
+ trafficConfiguration.setTenantSwitch(true);
+ trafficConfiguration.setGlobalSwitch(true);
trafficConfiguration.setMaxGlobalQpsRate(3);
RateLimitInterceptor rateLimitInterceptor = new RateLimitInterceptor(trafficConfiguration);
@@ -86,4 +86,4 @@ public class RateLimitInterceptorTest {
Assert.assertFalse(rateLimitInterceptor.preHandle(request, response, null));
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
index 12cdfe1e0d..cdc55df4c1 100644
--- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml
+++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
@@ -102,6 +102,20 @@ security:
user.identity.attribute: uid
user.email.attribute: mail
+# Traffic control, if you turn on this config, the maximum number of request/s will be limited.
+# global max request number per second
+# default tenant-level max request number
+traffic:
+ control:
+ global-switch: false
+ max-global-qps-rate: 300
+ tenant-switch: false
+ default-tenant-qps-rate: 10
+ #customize-tenant-qps-rate:
+ # eg.
+ #tenant1: 11
+ #tenant2: 20
+
master:
listen-port: 5678
# master fetch command num