You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2021/07/19 08:09:29 UTC

[skywalking] 01/01: Enhance persistent session timeout mechanism.

This is an automated email from the ASF dual-hosted git repository.

wusheng pushed a commit to branch metadata-ttl
in repository https://gitbox.apache.org/repos/asf/skywalking.git

commit 70a438166e0ab32c63522fa0ae4093d4394e6472
Author: Wu Sheng <wu...@foxmail.com>
AuthorDate: Mon Jul 19 16:09:09 2021 +0800

    Enhance persistent session timeout mechanism.
---
 CHANGES.md                                         |  3 ++
 .../oap/server/core/CoreModuleProvider.java        |  1 +
 .../analysis/worker/MetricsPersistentWorker.java   | 51 +++++++++++++++++++---
 .../analysis/worker/MetricsStreamProcessor.java    | 25 ++++++++---
 4 files changed, 67 insertions(+), 13 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 7c5019f..cebf8fd 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -13,6 +13,7 @@ Release Notes.
   consume.
 
 #### Java Agent
+
 * Supports modifying span attributes in async mode.
 * Agent supports the collection of JVM arguments and jar dependency information.
 * [Temporary] Support authentication for log report channel. This feature and grpc channel is going to be removed after
@@ -106,6 +107,8 @@ Release Notes.
 * Optimization: Concurrency mode of execution stage for metrics is removed(added in 8.5.0). Only concurrency of prepare
   stage is meaningful and kept.
 * Fix -meters metrics topic isn't created with namespace issue
+* Enhance persistent session timeout mechanism. Because the enhanced session could cache the metadata metrics forever,
+  new timeout mechanism is designed for avoiding this specific case.
 
 #### UI
 
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
index 31f5112..2731863 100755
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java
@@ -294,6 +294,7 @@ public class CoreModuleProvider extends ModuleProvider {
         metricsStreamProcessor.setEnableDatabaseSession(moduleConfig.isEnableDatabaseSession());
         metricsStreamProcessor.setL1FlushPeriod(moduleConfig.getL1FlushPeriod());
         metricsStreamProcessor.setStorageSessionTimeout(moduleConfig.getStorageSessionTimeout());
+        metricsStreamProcessor.setMetricsDataTTL(moduleConfig.getMetricsDataTTL());
         TopNStreamProcessor.getInstance().setTopNWorkerReportCycle(moduleConfig.getTopNReportPeriod());
         apdexThresholdConfig = new ApdexThresholdConfig(this);
         ApdexMetrics.setDICT(apdexThresholdConfig);
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentWorker.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentWorker.java
index 65e31a4..a528fff 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentWorker.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentWorker.java
@@ -33,6 +33,7 @@ import org.apache.skywalking.apm.commons.datacarrier.consumer.BulkConsumePool;
 import org.apache.skywalking.apm.commons.datacarrier.consumer.ConsumerPoolFactory;
 import org.apache.skywalking.apm.commons.datacarrier.consumer.IConsumer;
 import org.apache.skywalking.oap.server.core.UnexpectedException;
+import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
 import org.apache.skywalking.oap.server.core.analysis.data.MergableBufferedData;
 import org.apache.skywalking.oap.server.core.analysis.data.ReadWriteSafeCache;
 import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
@@ -78,11 +79,15 @@ public class MetricsPersistentWorker extends PersistenceWorker<Metrics> {
      * every {@link #persistentMod} periods. And minute level workers execute every time.
      */
     private int persistentMod;
+    /**
+     * @since 8.7.0 TTL settings from {@link org.apache.skywalking.oap.server.core.CoreModuleConfig#getMetricsDataTTL()}
+     */
+    private int metricsDataTTLInMillis;
 
     MetricsPersistentWorker(ModuleDefineHolder moduleDefineHolder, Model model, IMetricsDAO metricsDAO,
                             AbstractWorker<Metrics> nextAlarmWorker, AbstractWorker<ExportEvent> nextExportWorker,
                             MetricsTransWorker transWorker, boolean enableDatabaseSession, boolean supportUpdate,
-                            long storageSessionTimeout) {
+                            long storageSessionTimeout, int metricsDataTTL) {
         super(moduleDefineHolder, new ReadWriteSafeCache<>(new MergableBufferedData(), new MergableBufferedData()));
         this.model = model;
         this.context = new HashMap<>(100);
@@ -95,6 +100,8 @@ public class MetricsPersistentWorker extends PersistenceWorker<Metrics> {
         this.sessionTimeout = storageSessionTimeout;
         this.persistentCounter = 0;
         this.persistentMod = 1;
+        // Convert TTL to millis to make further calculation easier.
+        this.metricsDataTTLInMillis = metricsDataTTL * 24 * 60 * 60 * 1000;
 
         String name = "METRICS_L2_AGGREGATION";
         int size = BulkConsumePool.Creator.recommendMaxSize() / 8;
@@ -125,11 +132,16 @@ public class MetricsPersistentWorker extends PersistenceWorker<Metrics> {
     /**
      * Create the leaf and down-sampling MetricsPersistentWorker, no next step.
      */
-    MetricsPersistentWorker(ModuleDefineHolder moduleDefineHolder, Model model, IMetricsDAO metricsDAO,
-                            boolean enableDatabaseSession, boolean supportUpdate, long storageSessionTimeout) {
+    MetricsPersistentWorker(ModuleDefineHolder moduleDefineHolder,
+                            Model model,
+                            IMetricsDAO metricsDAO,
+                            boolean enableDatabaseSession,
+                            boolean supportUpdate,
+                            long storageSessionTimeout,
+                            int metricsDataTTL) {
         this(moduleDefineHolder, model, metricsDAO,
              null, null, null,
-             enableDatabaseSession, supportUpdate, storageSessionTimeout
+             enableDatabaseSession, supportUpdate, storageSessionTimeout, metricsDataTTL
         );
         // For a down-sampling metrics, we prolong the session timeout for 4 times, nearly 5 minutes.
         // And add offset according to worker creation sequence, to avoid context clear overlap,
@@ -246,10 +258,35 @@ public class MetricsPersistentWorker extends PersistenceWorker<Metrics> {
      * Load data from the storage, if {@link #enableDatabaseSession} == true, only load data when the id doesn't exist.
      */
     private void loadFromStorage(List<Metrics> metrics) {
+        final long currentTimeMillis = System.currentTimeMillis();
         try {
-            List<Metrics> notInCacheMetrics = metrics.stream()
-                                                     .filter(m -> !context.containsKey(m) || !enableDatabaseSession)
-                                                     .collect(Collectors.toList());
+            List<Metrics> notInCacheMetrics =
+                metrics.stream()
+                       .filter(m -> {
+                           final Metrics cachedValue = context.get(m);
+                           // Not cached or session disabled, the metric could be tagged `not in cache`.
+                           if (cachedValue == null || !enableDatabaseSession) {
+                               return true;
+                           }
+                           // The metric is in the cache, but still we have to check
+                           // whether the cache is expired due to TTL.
+                           // This is a cache-DB inconsistent case:
+                           // Metrics keep coming due to traffic, but the entity in the
+                           // database has been removed due to TTL.
+                           if (!model.isTimeRelativeID() && supportUpdate) {
+                               // Mostly all updatable metadata level metrics are required to do this check.
+                               final long metricsTimestamp = TimeBucket.getTimestamp(
+                                   cachedValue.getTimeBucket(), model.getDownsampling());
+                               // If the cached metric is older than the TTL indicated.
+                               if (currentTimeMillis - metricsTimestamp > metricsDataTTLInMillis) {
+                                   // The expired metrics should be tagged `not in cache` directly.
+                                   return true;
+                               }
+                           }
+
+                           return false;
+                       })
+                       .collect(Collectors.toList());
             if (notInCacheMetrics.isEmpty()) {
                 return;
             }
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java
index 6c323f4..1fb1d41 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java
@@ -87,6 +87,11 @@ public class MetricsStreamProcessor implements StreamProcessor<Metrics> {
      */
     @Setter
     private long storageSessionTimeout = 70_000;
+    /**
+     * @since 8.7.0 TTL settings from {@link org.apache.skywalking.oap.server.core.CoreModuleConfig#getMetricsDataTTL()}
+     */
+    @Setter
+    private int metricsDataTTL = 3;
 
     public static MetricsStreamProcessor getInstance() {
         return PROCESSOR;
@@ -157,12 +162,16 @@ public class MetricsStreamProcessor implements StreamProcessor<Metrics> {
         if (supportDownSampling) {
             if (configService.shouldToHour()) {
                 Model model = modelSetter.add(
-                    metricsClass, stream.getScopeId(), new Storage(stream.getName(), timeRelativeID, DownSampling.Hour), false);
+                    metricsClass, stream.getScopeId(), new Storage(stream.getName(), timeRelativeID, DownSampling.Hour),
+                    false
+                );
                 hourPersistentWorker = downSamplingWorker(moduleDefineHolder, metricsDAO, model, supportUpdate);
             }
             if (configService.shouldToDay()) {
                 Model model = modelSetter.add(
-                    metricsClass, stream.getScopeId(), new Storage(stream.getName(), timeRelativeID, DownSampling.Day), false);
+                    metricsClass, stream.getScopeId(), new Storage(stream.getName(), timeRelativeID, DownSampling.Day),
+                    false
+                );
                 dayPersistentWorker = downSamplingWorker(moduleDefineHolder, metricsDAO, model, supportUpdate);
             }
 
@@ -171,7 +180,9 @@ public class MetricsStreamProcessor implements StreamProcessor<Metrics> {
         }
 
         Model model = modelSetter.add(
-            metricsClass, stream.getScopeId(), new Storage(stream.getName(), timeRelativeID, DownSampling.Minute), false);
+            metricsClass, stream.getScopeId(), new Storage(stream.getName(), timeRelativeID, DownSampling.Minute),
+            false
+        );
         MetricsPersistentWorker minutePersistentWorker = minutePersistentWorker(
             moduleDefineHolder, metricsDAO, model, transWorker, supportUpdate);
 
@@ -197,8 +208,8 @@ public class MetricsStreamProcessor implements StreamProcessor<Metrics> {
         ExportWorker exportWorker = new ExportWorker(moduleDefineHolder);
 
         MetricsPersistentWorker minutePersistentWorker = new MetricsPersistentWorker(
-            moduleDefineHolder, model, metricsDAO, alarmNotifyWorker, exportWorker, transWorker, enableDatabaseSession,
-            supportUpdate, storageSessionTimeout
+            moduleDefineHolder, model, metricsDAO, alarmNotifyWorker, exportWorker, transWorker,
+            enableDatabaseSession, supportUpdate, storageSessionTimeout, metricsDataTTL
         );
         persistentWorkers.add(minutePersistentWorker);
 
@@ -210,7 +221,9 @@ public class MetricsStreamProcessor implements StreamProcessor<Metrics> {
                                                        Model model,
                                                        boolean supportUpdate) {
         MetricsPersistentWorker persistentWorker = new MetricsPersistentWorker(
-            moduleDefineHolder, model, metricsDAO, enableDatabaseSession, supportUpdate, storageSessionTimeout);
+            moduleDefineHolder, model, metricsDAO,
+            enableDatabaseSession, supportUpdate, storageSessionTimeout, metricsDataTTL
+        );
         persistentWorkers.add(persistentWorker);
 
         return persistentWorker;