You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2020/04/19 06:49:34 UTC

[skywalking] branch metrics updated: Change heatmap metrics group key logic. Fix more codes.

This is an automated email from the ASF dual-hosted git repository.

wusheng pushed a commit to branch metrics
in repository https://gitbox.apache.org/repos/asf/skywalking.git


The following commit(s) were added to refs/heads/metrics by this push:
     new b0ac1e8  Change heatmap metrics group key logic. Fix more codes.
b0ac1e8 is described below

commit b0ac1e87cde9ead2875ae5a4036602a61accc5c7
Author: Wu Sheng <wu...@foxmail.com>
AuthorDate: Sun Apr 19 14:48:46 2020 +0800

    Change heatmap metrics group key logic. Fix more codes.
---
 .../core/analysis/metrics/HistogramMetrics.java    |  6 +-
 .../oap/server/core/query/type/HeatMap.java        |  9 +--
 .../server/core/storage/model/StorageModels.java   |  3 +-
 .../core/analysis/metrics/DataTableTestCase.java   | 26 ++++-----
 ...ramMetricsTest.java => HeatMapMetricsTest.java} | 27 +++++----
 .../oap/query/graphql/resolver/MetricsQuery.java   | 10 ++++
 .../elasticsearch/query/MetricsQueryEsDAO.java     | 68 +++++++---------------
 .../elasticsearch7/query/MetricsQueryEs7DAO.java   | 50 ++++++++--------
 8 files changed, 88 insertions(+), 111 deletions(-)

diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetrics.java
index fd58da7..179f28c 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetrics.java
@@ -51,8 +51,8 @@ public abstract class HistogramMetrics extends Metrics {
     /**
      * Data will be grouped in
      * <pre>
-     * key = 100, represents [0, 100), value = count of requests in the latency range.
-     * key = 200, represents [100, 200), value = count of requests in the latency range.
+     * key = 0, represents [0, 100), value = count of requests in the latency range.
+     * key = 100, represents [100, 200), value = count of requests in the latency range.
      * ...
      * key = step * maxNumOfSteps, represents [step * maxNumOfSteps, MAX)
      * </pre>
@@ -69,7 +69,7 @@ public abstract class HistogramMetrics extends Metrics {
             }
         }
 
-        int index = value / step + 1;
+        int index = value / step;
         if (index > maxNumOfSteps) {
             index = maxNumOfSteps;
         }
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/HeatMap.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/HeatMap.java
index a5cb600..f33a855 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/HeatMap.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/HeatMap.java
@@ -53,12 +53,13 @@ public class HeatMap {
         if (buckets == null) {
             buckets = new ArrayList<>(dataset.size());
             for (int i = 0; i < sortedKeys.size(); i++) {
-                if (i == 0) {
-                    this.addBucket(new Bucket(0, Integer.parseInt(sortedKeys.get(i))));
+                if (i == sortedKeys.size() - 1) {
+                    // last element
+                    this.addBucket(new Bucket(Integer.parseInt(sortedKeys.get(i)), Integer.MAX_VALUE));
                 } else {
                     this.addBucket(new Bucket(
-                        Integer.parseInt(sortedKeys.get(i - 1)),
-                        Integer.parseInt(sortedKeys.get(i))
+                        Integer.parseInt(sortedKeys.get(i)),
+                        Integer.parseInt(sortedKeys.get(i + 1))
                     ));
                 }
             }
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
index d2c08cd..0a25e74 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
@@ -81,7 +81,8 @@ public class StorageModels implements IModelManager, INewModel, IModelOverride {
                     log.debug("The field named {} with the {} type", column.columnName(), field.getType());
                 }
                 if (column.isValue()) {
-                    ValueColumnMetadata.INSTANCE.putIfAbsent(modelName, column.columnName(), column.function());
+                    ValueColumnMetadata.INSTANCE.putIfAbsent(
+                        modelName, column.columnName(), column.function(), column.defaultValue());
                 }
 
                 List<QueryUnifiedIndex> indexDefinitions = new ArrayList<>();
diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTableTestCase.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTableTestCase.java
index f51d537..08b80af 100644
--- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTableTestCase.java
+++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTableTestCase.java
@@ -27,18 +27,12 @@ public class DataTableTestCase {
 
     @Before
     public void init() {
-        IntKeyLongValue v1 = new IntKeyLongValue(5, 500);
-        IntKeyLongValue v2 = new IntKeyLongValue(6, 600);
-        IntKeyLongValue v3 = new IntKeyLongValue(1, 100);
-        IntKeyLongValue v4 = new IntKeyLongValue(2, 200);
-        IntKeyLongValue v5 = new IntKeyLongValue(7, 700);
-
         dataTable = new DataTable();
-        dataTable.put(v1.getKey(), v1);
-        dataTable.put(v2.getKey(), v2);
-        dataTable.put(v3.getKey(), v3);
-        dataTable.put(v4.getKey(), v4);
-        dataTable.put(v5.getKey(), v5);
+        dataTable.put("5", 500L);
+        dataTable.put("6", 600L);
+        dataTable.put("1", 100L);
+        dataTable.put("2", 200L);
+        dataTable.put("7", 700L);
     }
 
     @Test
@@ -51,11 +45,11 @@ public class DataTableTestCase {
         DataTable dataTable = new DataTable();
         dataTable.toObject("1,100|2,200|5,500|6,600|7,700");
 
-        Assert.assertEquals(100, dataTable.get(1).getValue());
-        Assert.assertEquals(200, dataTable.get(2).getValue());
-        Assert.assertEquals(500, dataTable.get(5).getValue());
-        Assert.assertEquals(600, dataTable.get(6).getValue());
-        Assert.assertEquals(700, dataTable.get(7).getValue());
+        Assert.assertEquals(100, dataTable.get("1").intValue());
+        Assert.assertEquals(200, dataTable.get("2").intValue());
+        Assert.assertEquals(500, dataTable.get("5").intValue());
+        Assert.assertEquals(600, dataTable.get("6").intValue());
+        Assert.assertEquals(700, dataTable.get("7").intValue());
     }
 
     @Test
diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetricsTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/HeatMapMetricsTest.java
similarity index 82%
rename from oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetricsTest.java
rename to oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/HeatMapMetricsTest.java
index b683d5d..85e1c02 100644
--- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetricsTest.java
+++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/HeatMapMetricsTest.java
@@ -18,12 +18,11 @@
 
 package org.apache.skywalking.oap.server.core.analysis.metrics;
 
-import java.util.Map;
 import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
 import org.junit.Assert;
 import org.junit.Test;
 
-public class HistogramMetricsTest {
+public class HeatMapMetricsTest {
     private int step = 10; //ms
     private int maxNumOfSteps = 10; //count
 
@@ -45,13 +44,13 @@ public class HistogramMetricsTest {
         metricsMocker.combine(100, step, maxNumOfSteps);
         metricsMocker.combine(100, step, maxNumOfSteps);
 
-        Map<Integer, IntKeyLongValue> index = metricsMocker.getDataset();
-        Assert.assertEquals(4, index.size());
+        final DataTable dataset = metricsMocker.getDataset();
+        Assert.assertEquals(10, dataset.size());
 
-        Assert.assertEquals(1, index.get(2).getValue());
-        Assert.assertEquals(3, index.get(5).getValue());
-        Assert.assertEquals(1, index.get(6).getValue());
-        Assert.assertEquals(8, index.get(10).getValue());
+        Assert.assertEquals(1, dataset.get("20").intValue());
+        Assert.assertEquals(3, dataset.get("50").intValue());
+        Assert.assertEquals(1, dataset.get("60").intValue());
+        Assert.assertEquals(8, dataset.get("100").intValue());
     }
 
     @Test
@@ -77,13 +76,13 @@ public class HistogramMetricsTest {
 
         metricsMocker.combine(metricsMocker1);
 
-        Map<Integer, IntKeyLongValue> index = metricsMocker.getDataset();
-        Assert.assertEquals(4, index.size());
+        final DataTable dataset = metricsMocker.getDataset();
+        Assert.assertEquals(10, dataset.size());
 
-        Assert.assertEquals(1, index.get(2).getValue());
-        Assert.assertEquals(3, index.get(5).getValue());
-        Assert.assertEquals(1, index.get(6).getValue());
-        Assert.assertEquals(8, index.get(10).getValue());
+        Assert.assertEquals(1, dataset.get("20").intValue());
+        Assert.assertEquals(3, dataset.get("50").intValue());
+        Assert.assertEquals(1, dataset.get("60").intValue());
+        Assert.assertEquals(8, dataset.get("100").intValue());
     }
 
     public class HistogramMetricsMocker extends HistogramMetrics {
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetricsQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetricsQuery.java
index a4fb417..b450a3c 100644
--- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetricsQuery.java
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetricsQuery.java
@@ -117,6 +117,16 @@ public class MetricsQuery implements GraphQLQueryResolver {
 
     /**
      * Heatmap is bucket based value statistic result.
+     *
+     * @return heapmap including the latency distribution
+     * {@link HeatMap#getBuckets()}
+     * {@link HeatMap.HeatMapColumn#getValues()} follows this rule.
+     * <pre>
+     *      key = 0, represents [0, 100), value = count of requests in the latency range.
+     *      key = 100, represents [100, 200), value = count of requests in the latency range.
+     *      ...
+     *      key = step * maxNumOfSteps, represents [step * maxNumOfSteps, MAX)
+     * </pre>
      */
     public HeatMap readHeatMap(MetricsCondition condition, Duration duration) throws IOException {
         return getMetricsQueryService().readHeatMap(condition, duration);
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
index 84b6790..278e19d 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
@@ -23,10 +23,8 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import org.apache.skywalking.oap.server.core.analysis.DownSampling;
 import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
 import org.apache.skywalking.oap.server.core.analysis.metrics.HistogramMetrics;
-import org.apache.skywalking.oap.server.core.analysis.metrics.IntKeyLongValue;
 import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
 import org.apache.skywalking.oap.server.core.query.PointOfTime;
 import org.apache.skywalking.oap.server.core.query.input.Duration;
@@ -36,7 +34,6 @@ import org.apache.skywalking.oap.server.core.query.type.HeatMap;
 import org.apache.skywalking.oap.server.core.query.type.IntValues;
 import org.apache.skywalking.oap.server.core.query.type.KVInt;
 import org.apache.skywalking.oap.server.core.query.type.MetricsValues;
-import org.apache.skywalking.oap.server.core.query.type.Thermodynamic;
 import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
 import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO;
 import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
@@ -60,48 +57,6 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
     }
 
     @Override
-    public Thermodynamic getThermodynamic(String indexName, DownSampling downsampling, List<String> ids,
-                                          String valueCName) throws IOException {
-        Thermodynamic thermodynamic = new Thermodynamic();
-        List<List<Long>> thermodynamicValueMatrix = new ArrayList<>();
-
-        SearchResponse response = getClient().ids(indexName, ids.toArray(new String[0]));
-        Map<String, Map<String, Object>> idMap = toMap(response);
-
-        int numOfSteps = 0;
-        for (String id : ids) {
-            Map<String, Object> source = idMap.get(id);
-            if (source == null) {
-                // add empty list to represent no data exist for this time bucket
-                thermodynamicValueMatrix.add(new ArrayList<>());
-            } else {
-                int axisYStep = ((Number) source.get(HistogramMetrics.STEP)).intValue();
-                thermodynamic.setAxisYStep(axisYStep);
-                numOfSteps = ((Number) source.get(HistogramMetrics.NUM_OF_STEPS)).intValue() + 1;
-
-                String value = (String) source.get(HistogramMetrics.DATASET);
-                DataTable intKeyLongValues = new DataTable(5);
-                intKeyLongValues.toObject(value);
-
-                List<Long> axisYValues = new ArrayList<>();
-                for (int i = 0; i < numOfSteps; i++) {
-                    axisYValues.add(0L);
-                }
-
-                for (IntKeyLongValue intKeyLongValue : intKeyLongValues.values()) {
-                    axisYValues.set(intKeyLongValue.getKey(), intKeyLongValue.getValue());
-                }
-
-                thermodynamicValueMatrix.add(axisYValues);
-            }
-        }
-
-        thermodynamic.fromMatrixData(thermodynamicValueMatrix, numOfSteps);
-
-        return thermodynamic;
-    }
-
-    @Override
     public int readMetricsValue(final MetricsCondition condition,
                                 final String valueColumnName,
                                 final Duration duration) throws IOException {
@@ -221,7 +176,28 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
     public HeatMap readHeatMap(final MetricsCondition condition,
                                final String valueColumnName,
                                final Duration duration) throws IOException {
-        return null;
+        final List<PointOfTime> pointOfTimes = duration.assembleDurationPoints();
+        List<String> ids = new ArrayList<>(pointOfTimes.size());
+        pointOfTimes.forEach(pointOfTime -> {
+            ids.add(pointOfTime.id(condition.getEntity().buildId()));
+        });
+
+        SearchResponse response = getClient().ids(condition.getName(), ids.toArray(new String[0]));
+        Map<String, Map<String, Object>> idMap = toMap(response);
+
+        HeatMap heatMap = new HeatMap();
+
+        for (String id : ids) {
+            Map<String, Object> source = idMap.get(id);
+            if (source != null) {
+                String value = (String) source.get(HistogramMetrics.DATASET);
+                heatMap.buildColumn(id, value);
+            }
+        }
+
+        heatMap.fixMissingColumns(ids);
+
+        return heatMap;
     }
 
     protected void functionAggregation(Function function, TermsAggregationBuilder parentAggBuilder, String valueCName) {
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/MetricsQueryEs7DAO.java b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/MetricsQueryEs7DAO.java
index d1d9c59..977c3e0 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/MetricsQueryEs7DAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/MetricsQueryEs7DAO.java
@@ -19,12 +19,11 @@
 package org.apache.skywalking.oap.server.storage.plugin.elasticsearch7.query;
 
 import java.io.IOException;
-import org.apache.skywalking.oap.server.core.analysis.DownSampling;
 import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
-import org.apache.skywalking.oap.server.core.query.type.IntValues;
-import org.apache.skywalking.oap.server.core.query.type.KVInt;
+import org.apache.skywalking.oap.server.core.query.input.Duration;
+import org.apache.skywalking.oap.server.core.query.input.MetricsCondition;
 import org.apache.skywalking.oap.server.core.query.sql.Function;
-import org.apache.skywalking.oap.server.core.query.sql.Where;
+import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
 import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
 import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.MetricsQueryEsDAO;
 import org.elasticsearch.action.search.SearchResponse;
@@ -35,6 +34,11 @@ import org.elasticsearch.search.aggregations.metrics.Avg;
 import org.elasticsearch.search.aggregations.metrics.Sum;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
+/**
+ * Duplicate logic of {@link MetricsQueryEsDAO}, but for making compatible in ElasticSearch 7.
+ *
+ * {@link Avg} and {@link Sum} have different package in the es7.
+ */
 public class MetricsQueryEs7DAO extends MetricsQueryEsDAO {
 
     public MetricsQueryEs7DAO(ElasticSearchClient client) {
@@ -42,45 +46,37 @@ public class MetricsQueryEs7DAO extends MetricsQueryEsDAO {
     }
 
     @Override
-    public IntValues getValues(String indexName, DownSampling downsampling, long startTB, long endTB, Where where,
-                               String valueCName, Function function) throws IOException {
+    public int readMetricsValue(final MetricsCondition condition,
+                                final String valueColumnName,
+                                final Duration duration) throws IOException {
         SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
-        buildQuery(sourceBuilder, where, startTB, endTB);
+        buildQuery(sourceBuilder, condition, duration);
 
         TermsAggregationBuilder entityIdAggregation = AggregationBuilders.terms(Metrics.ENTITY_ID)
                                                                          .field(Metrics.ENTITY_ID)
-                                                                         .size(1000);
-        functionAggregation(function, entityIdAggregation, valueCName);
+                                                                         .size(1);
+        final Function function = ValueColumnMetadata.INSTANCE.getValueFunction(condition.getName());
+        functionAggregation(function, entityIdAggregation, valueColumnName);
 
         sourceBuilder.aggregation(entityIdAggregation);
 
-        SearchResponse response = getClient().search(indexName, sourceBuilder);
+        SearchResponse response = getClient().search(condition.getName(), sourceBuilder);
 
-        IntValues intValues = new IntValues();
         Terms idTerms = response.getAggregations().get(Metrics.ENTITY_ID);
         for (Terms.Bucket idBucket : idTerms.getBuckets()) {
-            long value;
             switch (function) {
                 case Sum:
-                    Sum sum = idBucket.getAggregations().get(valueCName);
-                    value = (long) sum.getValue();
-                    break;
+                    Sum sum = idBucket.getAggregations().get(valueColumnName);
+                    return (int) sum.getValue();
                 case Avg:
-                    Avg avg = idBucket.getAggregations().get(valueCName);
-                    value = (long) avg.getValue();
-                    break;
+                    Avg avg = idBucket.getAggregations().get(valueColumnName);
+                    return (int) avg.getValue();
                 default:
-                    avg = idBucket.getAggregations().get(valueCName);
-                    value = (long) avg.getValue();
-                    break;
+                    avg = idBucket.getAggregations().get(valueColumnName);
+                    return (int) avg.getValue();
             }
-
-            KVInt kvInt = new KVInt();
-            kvInt.setId(idBucket.getKeyAsString());
-            kvInt.setValue(value);
-            intValues.addKVInt(kvInt);
         }
-        return intValues;
+        return ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
     }
 
     protected void functionAggregation(Function function, TermsAggregationBuilder parentAggBuilder, String valueCName) {