You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2023/06/26 11:03:02 UTC
[skywalking] branch master updated: Control the period of training and sync HTTP URI recognition patterns (#10997)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 77da556c9e Control the period of training and sync HTTP URI recognition patterns (#10997)
77da556c9e is described below
commit 77da556c9e3f1adcde0b996ac926a53daad9d2ae
Author: 吴晟 Wu Sheng <wu...@foxmail.com>
AuthorDate: Mon Jun 26 19:02:56 2023 +0800
Control the period of training and sync HTTP URI recognition patterns (#10997)
---
docs/en/changes/changes.md | 9 +++++++--
docs/en/setup/backend/configuration-vocabulary.md | 2 ++
.../skywalking/oap/server/core/CoreModuleConfig.java | 13 +++++++++++++
.../skywalking/oap/server/core/CoreModuleProvider.java | 2 ++
.../server/core/config/group/EndpointNameGrouping.java | 15 +++++++++------
.../server-starter/src/main/resources/application.yml | 4 ++++
6 files changed, 37 insertions(+), 8 deletions(-)
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 5bd587794b..07d9cd7463 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -15,15 +15,20 @@
* Fix AI Pipeline uri caching NullPointer and IllegalArgument Exceptions.
* Fix `NPE` in metrics query when the metric is not exist.
* Remove E2E tests for Istio < 1.15, ElasticSearch < 7.16.3, they might still work but are not supported as planed.
-* Scroll all results in ElasticSearch storage and refactor scrolling logics, including Service, Instance, Endpoint, Process, etc.
+* Scroll all results in ElasticSearch storage and refactor scrolling logics, including Service, Instance, Endpoint,
+ Process, etc.
* Improve Kubernetes coordinator to remove `Terminating` OAP Pods in cluster.
+* Support `SW_CORE_SYNC_PERIOD_HTTP_URI_RECOGNITION_PATTERN` and `SW_CORE_TRAINING_PERIOD_HTTP_URI_RECOGNITION_PATTERN`
+ to control the period of training and sync HTTP URI recognition patterns. And shorten the default period to 10s for
+ sync and 60s for training.
#### UI
+
* Fix metric name `browser_app_error_rate` in `Browser-Root` dashboard.
#### Documentation
+
* Add Go agent into the server agent documentation.
* Add data unit description in the configuration of continuous profiling policy.
-
All issues and pull requests are [here](https://github.com/apache/skywalking/milestone/181?closed=1)
diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md
index b5e06c94c6..d27c07f5fb 100644
--- a/docs/en/setup/backend/configuration-vocabulary.md
+++ b/docs/en/setup/backend/configuration-vocabulary.md
@@ -83,6 +83,8 @@ The Configuration Vocabulary lists all available configurations provided by `app
| - | - | password | Nacos Auth password. [...]
| - | - | accessKey | Nacos Auth accessKey. [...]
| - | - | secretKey | Nacos Auth secretKey. [...]
+| - | - | syncPeriodHttpUriRecognitionPattern | The period of HTTP URI recognition pattern synchronization (in seconds). [...]
+| - | - | trainingPeriodHttpUriRecognitionPattern | The period of HTTP URI recognition pattern training (in seconds). [...]
| - | - | maxHttpUrisNumberPerService | The maximum number of HTTP URIs per service. [...]
| storage | elasticsearch | - | ElasticSearch (and OpenSearch) storage implementation. [...]
| - | - | namespace | Prefix of indexes created and used by SkyWalking. [...]
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java
index 2f7f74fdb3..40d036e9fa 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java
@@ -194,8 +194,21 @@ public class CoreModuleConfig extends ModuleConfig {
*/
private int httpMaxRequestHeaderSize = 8192;
+ /**
+ * The period of HTTP URI pattern recognition. Unit is second.
+ * @since 9.5.0
+ */
+ private int syncPeriodHttpUriRecognitionPattern = 10;
+
+ /**
+ * The training period of HTTP URI pattern recognition. Unit is second.
+ * @since 9.5.0
+ */
+ private int trainingPeriodHttpUriRecognitionPattern = 60;
+
/**
* The max number of HTTP URIs per service for further URI pattern recognition.
+ * @since 9.5.0
*/
private int maxHttpUrisNumberPerService = 3000;
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
index 0223213c75..8516ab2730 100755
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
@@ -357,6 +357,8 @@ public class CoreModuleProvider extends ModuleProvider {
.provider()
.getService(HttpUriRecognition.class),
getService(MetadataQueryService.class),
+ moduleConfig.getSyncPeriodHttpUriRecognitionPattern(),
+ moduleConfig.getTrainingPeriodHttpUriRecognitionPattern(),
moduleConfig.getMaxHttpUrisNumberPerService()
);
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java
index 8ac88b391e..c122db7fae 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java
@@ -91,7 +91,7 @@ public class EndpointNameGrouping {
formattedName = formatByQuickUriPattern(serviceName, endpointName);
ConcurrentHashMap<String, ArrayBlockingQueue<String>> svrHttpUris =
- cachedHttpUris.computeIfAbsent(serviceName, k -> new ConcurrentHashMap<>());
+ cachedHttpUris.computeIfAbsent(serviceName, k -> new ConcurrentHashMap<>());
// Only cache first N (determined by maxHttpUrisNumberPerService) URIs per 30 mins.
if (svrHttpUris.size() < maxHttpUrisNumberPerService) {
@@ -100,7 +100,7 @@ public class EndpointNameGrouping {
// code may accidentally retreive the size 1 queue created by unformatted endpoint
// The queue size is 10, which means only cache the first 10 formatted names.
final ArrayBlockingQueue<String> formattedURIs = svrHttpUris.computeIfAbsent(
- formattedName._1(), k -> new ArrayBlockingQueue<>(10));
+ formattedName._1(), k -> new ArrayBlockingQueue<>(10));
if (formattedURIs.size() < 10) {
// Try to push the raw URI as a candidate of formatted name.
formattedURIs.offer(endpointName);
@@ -160,6 +160,8 @@ public class EndpointNameGrouping {
public void startHttpUriRecognitionSvr(final HttpUriRecognition httpUriRecognitionSvr,
final MetadataQueryService metadataQueryService,
+ int syncPeriodHttpUriRecognitionPattern,
+ int trainingPeriodHttpUriRecognitionPattern,
int maxEndpointCandidatePerSvr) {
this.maxHttpUrisNumberPerService = maxEndpointCandidatePerSvr;
if (!httpUriRecognitionSvr.isInitialized()) {
@@ -170,8 +172,8 @@ public class EndpointNameGrouping {
.scheduleWithFixedDelay(
new RunnableWithExceptionProtection(
() -> {
- if (aiPipelineExecutionCounter.incrementAndGet() % 30 == 0) {
- // Send the cached URIs to the recognition server per 30 mins to build new patterns.
+ if (aiPipelineExecutionCounter.incrementAndGet() % trainingPeriodHttpUriRecognitionPattern == 0) {
+ // Send the cached URIs to the recognition server to build new patterns.
cachedHttpUris.forEach((serviceName, httpUris) -> {
final List<HttpUriRecognition.HTTPUri> candidates4UriPatterns = new ArrayList<>(
3000);
@@ -192,7 +194,8 @@ public class EndpointNameGrouping {
httpUris.clear();
httpUriRecognitionSvr.feedRawData(serviceName, candidates4UriPatterns);
});
- } else {
+ }
+ if (aiPipelineExecutionCounter.incrementAndGet() % syncPeriodHttpUriRecognitionPattern == 0) {
// Sync with the recognition server per 1 min to get the latest patterns.
try {
metadataQueryService.listServices(null, null).forEach(
@@ -216,7 +219,7 @@ public class EndpointNameGrouping {
}
},
t -> log.error("Fail to recognize URI patterns.", t)
- ), 1, 1, TimeUnit.MINUTES
+ ), 60, 1, TimeUnit.SECONDS
);
}
diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml
index fbaed52194..a4b334ca66 100644
--- a/oap-server/server-starter/src/main/resources/application.yml
+++ b/oap-server/server-starter/src/main/resources/application.yml
@@ -129,6 +129,10 @@ core:
prepareThreads: ${SW_CORE_PREPARE_THREADS:2}
# Turn it on then automatically grouping endpoint by the given OpenAPI definitions.
enableEndpointNameGroupingByOpenapi: ${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPENAPI:true}
+ # The period of HTTP URI pattern recognition. Unit is second.
+ syncPeriodHttpUriRecognitionPattern: ${SW_CORE_SYNC_PERIOD_HTTP_URI_RECOGNITION_PATTERN:10}
+ # The training period of HTTP URI pattern recognition. Unit is second.
+ trainingPeriodHttpUriRecognitionPattern: ${SW_CORE_TRAINING_PERIOD_HTTP_URI_RECOGNITION_PATTERN:60}
# The max number of HTTP URIs per service for further URI pattern recognition.
maxHttpUrisNumberPerService: ${SW_CORE_MAX_HTTP_URIS_NUMBER_PER_SVR:3000}
storage: