You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2020/03/31 01:04:18 UTC
[skywalking] branch master updated: Set the endpoint name length to
150, to keep safe and avoid over-length. (#4595)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 54fb6e7 Set the endpoint name length to 150, to keep safe and avoid over-length. (#4595)
54fb6e7 is described below
commit 54fb6e7285d695b1bd467edb6df0c3a3254adc12
Author: 吴晟 Wu Sheng <wu...@foxmail.com>
AuthorDate: Tue Mar 31 09:04:10 2020 +0800
Set the endpoint name length to 150, to keep safe and avoid over-length. (#4595)
* Set the endpoint name length to 150, to keep safe and avoid over-length.
* Set endpointNameMaxLength in the application.yml
* Fix a mischange.
* Correct the endpoint name length control logic
Co-authored-by: kezhenxu94 <ke...@apache.org>
---
.../skywalking/apm/agent/core/conf/Config.java | 20 ++++++++++----------
.../core/dictionary/NetworkAddressDictionary.java | 4 ++--
apm-sniffer/config/agent.config | 5 +++--
dist-material/application.yml | 4 +++-
.../src/main/resources/application.yml | 4 +++-
.../skywalking/oap/server/core/CoreModule.java | 22 +++++++++++++++++++++-
.../oap/server/core/CoreModuleConfig.java | 16 +++++++++-------
.../oap/server/core/CoreModuleProvider.java | 3 +++
.../oap/server/core/source/Endpoint.java | 7 ++++++-
.../oap/server/core/source/EndpointRelation.java | 17 ++++++++++++-----
10 files changed, 72 insertions(+), 30 deletions(-)
diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java
index 726aa4a..71fc41c 100755
--- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java
+++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java
@@ -18,14 +18,13 @@
package org.apache.skywalking.apm.agent.core.conf;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.skywalking.apm.agent.core.context.trace.TraceSegment;
import org.apache.skywalking.apm.agent.core.logging.core.LogLevel;
import org.apache.skywalking.apm.agent.core.logging.core.LogOutput;
import org.apache.skywalking.apm.agent.core.logging.core.WriterFactory;
-import java.util.HashMap;
-import java.util.Map;
-
/**
* This is the core config in sniffer agent.
*/
@@ -102,9 +101,12 @@ public class Config {
public static long FORCE_RECONNECTION_PERIOD = 1;
/**
- * Limit the length of the operationName to prevent errors when inserting elasticsearch
- **/
- public static int OPERATION_NAME_THRESHOLD = 500;
+ * Limit the length of the operationName to prevent the overlength issue in the storage.
+ *
+ * <p>NOTICE</p>
+ * In the current practice, we don't recommend the length over 190.
+ */
+ public static int OPERATION_NAME_THRESHOLD = 150;
}
public static class Collector {
@@ -173,11 +175,9 @@ public class Config {
public static class Dictionary {
/**
- * The buffer size of application codes and peer
+ * The buffer size of the registered network addresses.
*/
- public static int SERVICE_CODE_BUFFER_SIZE = 10 * 10000;
-
- public static int ENDPOINT_NAME_BUFFER_SIZE = 1000 * 10000;
+ public static int NETWORK_ADDRESS_BUFFER_SIZE = 10 * 10000;
}
public static class Logging {
diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/dictionary/NetworkAddressDictionary.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/dictionary/NetworkAddressDictionary.java
index 99f187d..ac30860 100755
--- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/dictionary/NetworkAddressDictionary.java
+++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/dictionary/NetworkAddressDictionary.java
@@ -26,7 +26,7 @@ import org.apache.skywalking.apm.network.register.v2.NetAddressMapping;
import org.apache.skywalking.apm.network.register.v2.NetAddresses;
import org.apache.skywalking.apm.network.register.v2.RegisterGrpc;
-import static org.apache.skywalking.apm.agent.core.conf.Config.Dictionary.SERVICE_CODE_BUFFER_SIZE;
+import static org.apache.skywalking.apm.agent.core.conf.Config.Dictionary.NETWORK_ADDRESS_BUFFER_SIZE;
/**
* Map of network address id to network literal address, which is from the collector side.
@@ -41,7 +41,7 @@ public enum NetworkAddressDictionary {
if (applicationId != null) {
return new Found(applicationId);
} else {
- if (serviceDictionary.size() + unRegisterServices.size() < SERVICE_CODE_BUFFER_SIZE) {
+ if (serviceDictionary.size() + unRegisterServices.size() < NETWORK_ADDRESS_BUFFER_SIZE) {
unRegisterServices.add(networkAddress);
}
return new NotFound();
diff --git a/apm-sniffer/config/agent.config b/apm-sniffer/config/agent.config
index ee6dc5a..23cb92a 100644
--- a/apm-sniffer/config/agent.config
+++ b/apm-sniffer/config/agent.config
@@ -29,7 +29,7 @@ agent.service_name=${SW_AGENT_NAME:Your_ApplicationName}
# The max amount of spans in a single segment.
# Through this config item, SkyWalking keep your application memory cost estimated.
-# agent.span_limit_per_segment=${SW_AGENT_SPAN_LIMIT:300}
+# agent.span_limit_per_segment=${SW_AGENT_SPAN_LIMIT:150}
# Ignore the segments if their operation names end with these suffix.
# agent.ignore_suffix=${SW_AGENT_IGNORE_SUFFIX:.jpg,.jpeg,.js,.css,.png,.bmp,.gif,.ico,.mp3,.mp4,.html,.svg}
@@ -39,7 +39,8 @@ agent.service_name=${SW_AGENT_NAME:Your_ApplicationName}
# agent.is_open_debugging_class = ${SW_AGENT_OPEN_DEBUG:true}
# The operationName max length
-# agent.operation_name_threshold=${SW_AGENT_OPERATION_NAME_THRESHOLD:500}
+# Notice, in the current practice, we don't recommend the length over 190.
+# agent.operation_name_threshold=${SW_AGENT_OPERATION_NAME_THRESHOLD:150}
# If true, skywalking agent will enable profile when user create a new profile task. Otherwise disable profile.
# profile.active=${SW_AGENT_PROFILE_ACTIVE:true}
diff --git a/dist-material/application.yml b/dist-material/application.yml
index f715df9..5e38bfc 100644
--- a/dist-material/application.yml
+++ b/dist-material/application.yml
@@ -85,7 +85,9 @@ core:
# and it will cause more load for memory, network of OAP and storage.
# But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party tool, such as Kibana->ES, to query the data by themselves.
activeExtraModelColumns: ${SW_CORE_ACTIVE_EXTRA_MODEL_COLUMNS:false}
-
+ # The max length of the endpoint name.
+ # In the current practice, we don't recommend the length over 190.
+ endpointNameMaxLength: ${SW_CORE_ENDPOINT_NAME_MAX_LENGTH:150}
storage:
selector: ${SW_STORAGE:h2}
elasticsearch:
diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml
index 7c79625..c794452 100755
--- a/oap-server/server-bootstrap/src/main/resources/application.yml
+++ b/oap-server/server-bootstrap/src/main/resources/application.yml
@@ -84,7 +84,9 @@ core:
# and it will cause more load for memory, network of OAP and storage.
# But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party tool, such as Kibana->ES, to query the data by themselves.
activeExtraModelColumns: ${SW_CORE_ACTIVE_EXTRA_MODEL_COLUMNS:false}
-
+ # The max length of the endpoint name.
+ # In the current practice, we don't recommend the length over 190.
+ endpointNameMaxLength: ${SW_CORE_ENDPOINT_NAME_MAX_LENGTH:150}
storage:
selector: ${SW_STORAGE:h2}
elasticsearch:
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java
index 81878e6..0dcd045 100755
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java
@@ -57,8 +57,8 @@ import org.apache.skywalking.oap.server.library.module.ModuleDefine;
* Core module definition. Define all open services to other modules.
*/
public class CoreModule extends ModuleDefine {
-
public static final String NAME = "core";
+ private static int ENDPOINT_NAME_MAX_LENGTH = 150;
public CoreModule() {
super(NAME);
@@ -87,6 +87,26 @@ public class CoreModule extends ModuleDefine {
return classes.toArray(new Class[] {});
}
+ /**
+ * Format endpoint name by using the length config in the core module. This is a global rule, every place including
+ * endpoint as the {@link org.apache.skywalking.oap.server.core.source.Source} should follow this for any core
+ * module implementation.
+ *
+ * @param endpointName raw data, literal string.
+ * @return the string, which length less than or equals {@link #ENDPOINT_NAME_MAX_LENGTH};
+ */
+ public static String formatEndpointName(String endpointName) {
+ if (endpointName.length() > ENDPOINT_NAME_MAX_LENGTH) {
+ return endpointName.substring(0, ENDPOINT_NAME_MAX_LENGTH);
+ } else {
+ return endpointName;
+ }
+ }
+
+ public static void setEndpointNameMaxLength(final int endpointNameMaxLength) {
+ ENDPOINT_NAME_MAX_LENGTH = endpointNameMaxLength;
+ }
+
private void addProfileService(List<Class> classes) {
classes.add(ProfileTaskMutationService.class);
classes.add(ProfileTaskQueryService.class);
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java
index e59ae44..96b8fb5 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java
@@ -88,7 +88,6 @@ public class CoreModuleConfig extends ModuleConfig {
*/
@Setter
private int remoteTimeout = 20;
-
/**
* Following are cache settings for inventory(s)
*/
@@ -96,31 +95,34 @@ public class CoreModuleConfig extends ModuleConfig {
private long maxSizeOfServiceInstanceInventory = 1_000_000L;
private long maxSizeOfEndpointInventory = 1_000_000L;
private long maxSizeOfNetworkInventory = 1_000_000L;
-
/**
* Following are cache setting for none stream(s)
*/
private long maxSizeOfProfileTask = 10_000L;
-
/**
* Analyze profile snapshots paging size.
*/
private int maxPageSizeOfQueryProfileSnapshot = 500;
-
/**
* Analyze profile snapshots max size.
*/
private int maxSizeOfAnalyzeProfileSnapshot = 12000;
-
/**
* Extra model column are the column defined by {@link ScopeDefaultColumn.DefinedByField#requireDynamicActive()} ==
- * true. These columns of model are not required logically in aggregation or further query, and it will cause more load for
- * memory, network of OAP and storage.
+ * true. These columns of model are not required logically in aggregation or further query, and it will cause more
+ * load for memory, network of OAP and storage.
*
* But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party
* tool, such as Kibana->ES, to query the data by themselves.
*/
private boolean activeExtraModelColumns = false;
+ /**
+ * The max length of the endpoint name.
+ *
+ * <p>NOTICE</p>
+ * In the current practice, we don't recommend the length over 190.
+ */
+ private int endpointNameMaxLength = 150;
public CoreModuleConfig() {
this.downsampling = new ArrayList<>();
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
index 547c57d..43d90c8 100755
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
@@ -139,6 +139,9 @@ public class CoreModuleProvider extends ModuleProvider {
if (moduleConfig.isActiveExtraModelColumns()) {
DefaultScopeDefine.activeExtraModelColumns();
}
+ if (moduleConfig.getEndpointNameMaxLength() > 0) {
+ CoreModule.setEndpointNameMaxLength(moduleConfig.getEndpointNameMaxLength());
+ }
StreamAnnotationListener streamAnnotationListener = new StreamAnnotationListener(getManager());
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java
index 821f768..9ce9186 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java
@@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.source;
import lombok.Getter;
import lombok.Setter;
+import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic;
import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ENDPOINT;
@@ -42,9 +43,13 @@ public class Endpoint extends Source {
}
@Getter
- @Setter
@ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
private String name;
+
+ public void setName(final String name) {
+ this.name = CoreModule.formatEndpointName(name);
+ }
+
@Getter
@Setter
@ScopeDefaultColumn.DefinedByField(columnName = "service_id")
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EndpointRelation.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EndpointRelation.java
index 4536929..473120d 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EndpointRelation.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EndpointRelation.java
@@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.source;
import lombok.Getter;
import lombok.Setter;
+import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.analysis.manual.RelationDefineUtil;
import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ENDPOINT_RELATION;
@@ -44,9 +45,13 @@ public class EndpointRelation extends Source {
}
@Getter
- @Setter
@ScopeDefaultColumn.DefinedByField(columnName = "source_endpoint_name")
private String endpoint;
+
+ public void setEndpoint(final String endpoint) {
+ this.endpoint = CoreModule.formatEndpointName(endpoint);
+ }
+
@Getter
@Setter
@ScopeDefaultColumn.DefinedByField(columnName = "service_id")
@@ -61,17 +66,20 @@ public class EndpointRelation extends Source {
@Getter
@Setter
private String serviceInstanceName;
-
@Getter
- @Setter
@ScopeDefaultColumn.DefinedByField(columnName = "child_endpoint_name")
private String childEndpoint;
+
+ public void setChildEndpoint(final String childEndpoint) {
+ this.childEndpoint = CoreModule.formatEndpointName(childEndpoint);
+ }
+
@Getter
@Setter
@ScopeDefaultColumn.DefinedByField(columnName = "child_service_id")
private int childServiceId;
- @Getter
@Setter
+ @Getter
@ScopeDefaultColumn.DefinedByField(columnName = "child_service_name", requireDynamicActive = true)
private String childServiceName;
@Getter
@@ -80,7 +88,6 @@ public class EndpointRelation extends Source {
@Getter
@Setter
private String childServiceInstanceName;
-
@Getter
@Setter
private int componentId;