You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2021/07/19 08:09:29 UTC
[skywalking] 01/01: Enhance persistent session timeout mechanism.
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch metadata-ttl
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 70a438166e0ab32c63522fa0ae4093d4394e6472
Author: Wu Sheng <wu...@foxmail.com>
AuthorDate: Mon Jul 19 16:09:09 2021 +0800
Enhance persistent session timeout mechanism.
---
CHANGES.md | 3 ++
.../oap/server/core/CoreModuleProvider.java | 1 +
.../analysis/worker/MetricsPersistentWorker.java | 51 +++++++++++++++++++---
.../analysis/worker/MetricsStreamProcessor.java | 25 ++++++++---
4 files changed, 67 insertions(+), 13 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 7c5019f..cebf8fd 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -13,6 +13,7 @@ Release Notes.
consume.
#### Java Agent
+
* Supports modifying span attributes in async mode.
* Agent supports the collection of JVM arguments and jar dependency information.
* [Temporary] Support authentication for log report channel. This feature and grpc channel is going to be removed after
@@ -106,6 +107,8 @@ Release Notes.
* Optimization: Concurrency mode of execution stage for metrics is removed(added in 8.5.0). Only concurrency of prepare
stage is meaningful and kept.
* Fix -meters metrics topic isn't created with namespace issue
+* Enhance persistent session timeout mechanism. Because the enhanced session could cache the metadata metrics forever,
+ new timeout mechanism is designed for avoiding this specific case.
#### UI
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
index 31f5112..2731863 100755
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
@@ -294,6 +294,7 @@ public class CoreModuleProvider extends ModuleProvider {
metricsStreamProcessor.setEnableDatabaseSession(moduleConfig.isEnableDatabaseSession());
metricsStreamProcessor.setL1FlushPeriod(moduleConfig.getL1FlushPeriod());
metricsStreamProcessor.setStorageSessionTimeout(moduleConfig.getStorageSessionTimeout());
+ metricsStreamProcessor.setMetricsDataTTL(moduleConfig.getMetricsDataTTL());
TopNStreamProcessor.getInstance().setTopNWorkerReportCycle(moduleConfig.getTopNReportPeriod());
apdexThresholdConfig = new ApdexThresholdConfig(this);
ApdexMetrics.setDICT(apdexThresholdConfig);
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentWorker.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentWorker.java
index 65e31a4..a528fff 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentWorker.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentWorker.java
@@ -33,6 +33,7 @@ import org.apache.skywalking.apm.commons.datacarrier.consumer.BulkConsumePool;
import org.apache.skywalking.apm.commons.datacarrier.consumer.ConsumerPoolFactory;
import org.apache.skywalking.apm.commons.datacarrier.consumer.IConsumer;
import org.apache.skywalking.oap.server.core.UnexpectedException;
+import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import org.apache.skywalking.oap.server.core.analysis.data.MergableBufferedData;
import org.apache.skywalking.oap.server.core.analysis.data.ReadWriteSafeCache;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
@@ -78,11 +79,15 @@ public class MetricsPersistentWorker extends PersistenceWorker<Metrics> {
* every {@link #persistentMod} periods. And minute level workers execute every time.
*/
private int persistentMod;
+ /**
+ * @since 8.7.0 TTL settings from {@link org.apache.skywalking.oap.server.core.CoreModuleConfig#getMetricsDataTTL()}
+ */
+ private int metricsDataTTLInMillis;
MetricsPersistentWorker(ModuleDefineHolder moduleDefineHolder, Model model, IMetricsDAO metricsDAO,
AbstractWorker<Metrics> nextAlarmWorker, AbstractWorker<ExportEvent> nextExportWorker,
MetricsTransWorker transWorker, boolean enableDatabaseSession, boolean supportUpdate,
- long storageSessionTimeout) {
+ long storageSessionTimeout, int metricsDataTTL) {
super(moduleDefineHolder, new ReadWriteSafeCache<>(new MergableBufferedData(), new MergableBufferedData()));
this.model = model;
this.context = new HashMap<>(100);
@@ -95,6 +100,8 @@ public class MetricsPersistentWorker extends PersistenceWorker<Metrics> {
this.sessionTimeout = storageSessionTimeout;
this.persistentCounter = 0;
this.persistentMod = 1;
+ // Convert TTL to millis to make further calculation easier.
+ this.metricsDataTTLInMillis = metricsDataTTL * 24 * 60 * 60 * 1000;
String name = "METRICS_L2_AGGREGATION";
int size = BulkConsumePool.Creator.recommendMaxSize() / 8;
@@ -125,11 +132,16 @@ public class MetricsPersistentWorker extends PersistenceWorker<Metrics> {
/**
* Create the leaf and down-sampling MetricsPersistentWorker, no next step.
*/
- MetricsPersistentWorker(ModuleDefineHolder moduleDefineHolder, Model model, IMetricsDAO metricsDAO,
- boolean enableDatabaseSession, boolean supportUpdate, long storageSessionTimeout) {
+ MetricsPersistentWorker(ModuleDefineHolder moduleDefineHolder,
+ Model model,
+ IMetricsDAO metricsDAO,
+ boolean enableDatabaseSession,
+ boolean supportUpdate,
+ long storageSessionTimeout,
+ int metricsDataTTL) {
this(moduleDefineHolder, model, metricsDAO,
null, null, null,
- enableDatabaseSession, supportUpdate, storageSessionTimeout
+ enableDatabaseSession, supportUpdate, storageSessionTimeout, metricsDataTTL
);
// For a down-sampling metrics, we prolong the session timeout for 4 times, nearly 5 minutes.
// And add offset according to worker creation sequence, to avoid context clear overlap,
@@ -246,10 +258,35 @@ public class MetricsPersistentWorker extends PersistenceWorker<Metrics> {
* Load data from the storage, if {@link #enableDatabaseSession} == true, only load data when the id doesn't exist.
*/
private void loadFromStorage(List<Metrics> metrics) {
+ final long currentTimeMillis = System.currentTimeMillis();
try {
- List<Metrics> notInCacheMetrics = metrics.stream()
- .filter(m -> !context.containsKey(m) || !enableDatabaseSession)
- .collect(Collectors.toList());
+ List<Metrics> notInCacheMetrics =
+ metrics.stream()
+ .filter(m -> {
+ final Metrics cachedValue = context.get(m);
+ // Not cached or session disabled, the metric could be tagged `not in cache`.
+ if (cachedValue == null || !enableDatabaseSession) {
+ return true;
+ }
+ // The metric is in the cache, but still we have to check
+ // whether the cache is expired due to TTL.
+ // This is a cache-DB inconsistent case:
+ // Metrics keep coming due to traffic, but the entity in the
+ // database has been removed due to TTL.
+ if (!model.isTimeRelativeID() && supportUpdate) {
+ // Mostly all updatable metadata level metrics are required to do this check.
+ final long metricsTimestamp = TimeBucket.getTimestamp(
+ cachedValue.getTimeBucket(), model.getDownsampling());
+ // If the cached metric is older than the TTL indicated.
+ if (currentTimeMillis - metricsTimestamp > metricsDataTTLInMillis) {
+ // The expired metrics should be tagged `not in cache` directly.
+ return true;
+ }
+ }
+
+ return false;
+ })
+ .collect(Collectors.toList());
if (notInCacheMetrics.isEmpty()) {
return;
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java
index 6c323f4..1fb1d41 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java
@@ -87,6 +87,11 @@ public class MetricsStreamProcessor implements StreamProcessor<Metrics> {
*/
@Setter
private long storageSessionTimeout = 70_000;
+ /**
+ * @since 8.7.0 TTL settings from {@link org.apache.skywalking.oap.server.core.CoreModuleConfig#getMetricsDataTTL()}
+ */
+ @Setter
+ private int metricsDataTTL = 3;
public static MetricsStreamProcessor getInstance() {
return PROCESSOR;
@@ -157,12 +162,16 @@ public class MetricsStreamProcessor implements StreamProcessor<Metrics> {
if (supportDownSampling) {
if (configService.shouldToHour()) {
Model model = modelSetter.add(
- metricsClass, stream.getScopeId(), new Storage(stream.getName(), timeRelativeID, DownSampling.Hour), false);
+ metricsClass, stream.getScopeId(), new Storage(stream.getName(), timeRelativeID, DownSampling.Hour),
+ false
+ );
hourPersistentWorker = downSamplingWorker(moduleDefineHolder, metricsDAO, model, supportUpdate);
}
if (configService.shouldToDay()) {
Model model = modelSetter.add(
- metricsClass, stream.getScopeId(), new Storage(stream.getName(), timeRelativeID, DownSampling.Day), false);
+ metricsClass, stream.getScopeId(), new Storage(stream.getName(), timeRelativeID, DownSampling.Day),
+ false
+ );
dayPersistentWorker = downSamplingWorker(moduleDefineHolder, metricsDAO, model, supportUpdate);
}
@@ -171,7 +180,9 @@ public class MetricsStreamProcessor implements StreamProcessor<Metrics> {
}
Model model = modelSetter.add(
- metricsClass, stream.getScopeId(), new Storage(stream.getName(), timeRelativeID, DownSampling.Minute), false);
+ metricsClass, stream.getScopeId(), new Storage(stream.getName(), timeRelativeID, DownSampling.Minute),
+ false
+ );
MetricsPersistentWorker minutePersistentWorker = minutePersistentWorker(
moduleDefineHolder, metricsDAO, model, transWorker, supportUpdate);
@@ -197,8 +208,8 @@ public class MetricsStreamProcessor implements StreamProcessor<Metrics> {
ExportWorker exportWorker = new ExportWorker(moduleDefineHolder);
MetricsPersistentWorker minutePersistentWorker = new MetricsPersistentWorker(
- moduleDefineHolder, model, metricsDAO, alarmNotifyWorker, exportWorker, transWorker, enableDatabaseSession,
- supportUpdate, storageSessionTimeout
+ moduleDefineHolder, model, metricsDAO, alarmNotifyWorker, exportWorker, transWorker,
+ enableDatabaseSession, supportUpdate, storageSessionTimeout, metricsDataTTL
);
persistentWorkers.add(minutePersistentWorker);
@@ -210,7 +221,9 @@ public class MetricsStreamProcessor implements StreamProcessor<Metrics> {
Model model,
boolean supportUpdate) {
MetricsPersistentWorker persistentWorker = new MetricsPersistentWorker(
- moduleDefineHolder, model, metricsDAO, enableDatabaseSession, supportUpdate, storageSessionTimeout);
+ moduleDefineHolder, model, metricsDAO,
+ enableDatabaseSession, supportUpdate, storageSessionTimeout, metricsDataTTL
+ );
persistentWorkers.add(persistentWorker);
return persistentWorker;