You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2020/04/19 06:49:34 UTC
[skywalking] branch metrics updated: Change heatmap metrics group
key logic. Fix more codes.
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch metrics
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/metrics by this push:
new b0ac1e8 Change heatmap metrics group key logic. Fix more codes.
b0ac1e8 is described below
commit b0ac1e87cde9ead2875ae5a4036602a61accc5c7
Author: Wu Sheng <wu...@foxmail.com>
AuthorDate: Sun Apr 19 14:48:46 2020 +0800
Change heatmap metrics group key logic. Fix more codes.
---
.../core/analysis/metrics/HistogramMetrics.java | 6 +-
.../oap/server/core/query/type/HeatMap.java | 9 +--
.../server/core/storage/model/StorageModels.java | 3 +-
.../core/analysis/metrics/DataTableTestCase.java | 26 ++++-----
...ramMetricsTest.java => HeatMapMetricsTest.java} | 27 +++++----
.../oap/query/graphql/resolver/MetricsQuery.java | 10 ++++
.../elasticsearch/query/MetricsQueryEsDAO.java | 68 +++++++---------------
.../elasticsearch7/query/MetricsQueryEs7DAO.java | 50 ++++++++--------
8 files changed, 88 insertions(+), 111 deletions(-)
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetrics.java
index fd58da7..179f28c 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetrics.java
@@ -51,8 +51,8 @@ public abstract class HistogramMetrics extends Metrics {
/**
* Data will be grouped in
* <pre>
- * key = 100, represents [0, 100), value = count of requests in the latency range.
- * key = 200, represents [100, 200), value = count of requests in the latency range.
+ * key = 0, represents [0, 100), value = count of requests in the latency range.
+ * key = 100, represents [100, 200), value = count of requests in the latency range.
* ...
* key = step * maxNumOfSteps, represents [step * maxNumOfSteps, MAX)
* </pre>
@@ -69,7 +69,7 @@ public abstract class HistogramMetrics extends Metrics {
}
}
- int index = value / step + 1;
+ int index = value / step;
if (index > maxNumOfSteps) {
index = maxNumOfSteps;
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/HeatMap.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/HeatMap.java
index a5cb600..f33a855 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/HeatMap.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/HeatMap.java
@@ -53,12 +53,13 @@ public class HeatMap {
if (buckets == null) {
buckets = new ArrayList<>(dataset.size());
for (int i = 0; i < sortedKeys.size(); i++) {
- if (i == 0) {
- this.addBucket(new Bucket(0, Integer.parseInt(sortedKeys.get(i))));
+ if (i == sortedKeys.size() - 1) {
+ // last element
+ this.addBucket(new Bucket(Integer.parseInt(sortedKeys.get(i)), Integer.MAX_VALUE));
} else {
this.addBucket(new Bucket(
- Integer.parseInt(sortedKeys.get(i - 1)),
- Integer.parseInt(sortedKeys.get(i))
+ Integer.parseInt(sortedKeys.get(i)),
+ Integer.parseInt(sortedKeys.get(i + 1))
));
}
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
index d2c08cd..0a25e74 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
@@ -81,7 +81,8 @@ public class StorageModels implements IModelManager, INewModel, IModelOverride {
log.debug("The field named {} with the {} type", column.columnName(), field.getType());
}
if (column.isValue()) {
- ValueColumnMetadata.INSTANCE.putIfAbsent(modelName, column.columnName(), column.function());
+ ValueColumnMetadata.INSTANCE.putIfAbsent(
+ modelName, column.columnName(), column.function(), column.defaultValue());
}
List<QueryUnifiedIndex> indexDefinitions = new ArrayList<>();
diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTableTestCase.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTableTestCase.java
index f51d537..08b80af 100644
--- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTableTestCase.java
+++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTableTestCase.java
@@ -27,18 +27,12 @@ public class DataTableTestCase {
@Before
public void init() {
- IntKeyLongValue v1 = new IntKeyLongValue(5, 500);
- IntKeyLongValue v2 = new IntKeyLongValue(6, 600);
- IntKeyLongValue v3 = new IntKeyLongValue(1, 100);
- IntKeyLongValue v4 = new IntKeyLongValue(2, 200);
- IntKeyLongValue v5 = new IntKeyLongValue(7, 700);
-
dataTable = new DataTable();
- dataTable.put(v1.getKey(), v1);
- dataTable.put(v2.getKey(), v2);
- dataTable.put(v3.getKey(), v3);
- dataTable.put(v4.getKey(), v4);
- dataTable.put(v5.getKey(), v5);
+ dataTable.put("5", 500L);
+ dataTable.put("6", 600L);
+ dataTable.put("1", 100L);
+ dataTable.put("2", 200L);
+ dataTable.put("7", 700L);
}
@Test
@@ -51,11 +45,11 @@ public class DataTableTestCase {
DataTable dataTable = new DataTable();
dataTable.toObject("1,100|2,200|5,500|6,600|7,700");
- Assert.assertEquals(100, dataTable.get(1).getValue());
- Assert.assertEquals(200, dataTable.get(2).getValue());
- Assert.assertEquals(500, dataTable.get(5).getValue());
- Assert.assertEquals(600, dataTable.get(6).getValue());
- Assert.assertEquals(700, dataTable.get(7).getValue());
+ Assert.assertEquals(100, dataTable.get("1").intValue());
+ Assert.assertEquals(200, dataTable.get("2").intValue());
+ Assert.assertEquals(500, dataTable.get("5").intValue());
+ Assert.assertEquals(600, dataTable.get("6").intValue());
+ Assert.assertEquals(700, dataTable.get("7").intValue());
}
@Test
diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetricsTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/HeatMapMetricsTest.java
similarity index 82%
rename from oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetricsTest.java
rename to oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/HeatMapMetricsTest.java
index b683d5d..85e1c02 100644
--- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetricsTest.java
+++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/metrics/HeatMapMetricsTest.java
@@ -18,12 +18,11 @@
package org.apache.skywalking.oap.server.core.analysis.metrics;
-import java.util.Map;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.junit.Assert;
import org.junit.Test;
-public class HistogramMetricsTest {
+public class HeatMapMetricsTest {
private int step = 10; //ms
private int maxNumOfSteps = 10; //count
@@ -45,13 +44,13 @@ public class HistogramMetricsTest {
metricsMocker.combine(100, step, maxNumOfSteps);
metricsMocker.combine(100, step, maxNumOfSteps);
- Map<Integer, IntKeyLongValue> index = metricsMocker.getDataset();
- Assert.assertEquals(4, index.size());
+ final DataTable dataset = metricsMocker.getDataset();
+ Assert.assertEquals(10, dataset.size());
- Assert.assertEquals(1, index.get(2).getValue());
- Assert.assertEquals(3, index.get(5).getValue());
- Assert.assertEquals(1, index.get(6).getValue());
- Assert.assertEquals(8, index.get(10).getValue());
+ Assert.assertEquals(1, dataset.get("20").intValue());
+ Assert.assertEquals(3, dataset.get("50").intValue());
+ Assert.assertEquals(1, dataset.get("60").intValue());
+ Assert.assertEquals(8, dataset.get("100").intValue());
}
@Test
@@ -77,13 +76,13 @@ public class HistogramMetricsTest {
metricsMocker.combine(metricsMocker1);
- Map<Integer, IntKeyLongValue> index = metricsMocker.getDataset();
- Assert.assertEquals(4, index.size());
+ final DataTable dataset = metricsMocker.getDataset();
+ Assert.assertEquals(10, dataset.size());
- Assert.assertEquals(1, index.get(2).getValue());
- Assert.assertEquals(3, index.get(5).getValue());
- Assert.assertEquals(1, index.get(6).getValue());
- Assert.assertEquals(8, index.get(10).getValue());
+ Assert.assertEquals(1, dataset.get("20").intValue());
+ Assert.assertEquals(3, dataset.get("50").intValue());
+ Assert.assertEquals(1, dataset.get("60").intValue());
+ Assert.assertEquals(8, dataset.get("100").intValue());
}
public class HistogramMetricsMocker extends HistogramMetrics {
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetricsQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetricsQuery.java
index a4fb417..b450a3c 100644
--- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetricsQuery.java
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetricsQuery.java
@@ -117,6 +117,16 @@ public class MetricsQuery implements GraphQLQueryResolver {
/**
* Heatmap is bucket based value statistic result.
+ *
+ * @return heapmap including the latency distribution
+ * {@link HeatMap#getBuckets()}
+ * {@link HeatMap.HeatMapColumn#getValues()} follows this rule.
+ * <pre>
+ * key = 0, represents [0, 100), value = count of requests in the latency range.
+ * key = 100, represents [100, 200), value = count of requests in the latency range.
+ * ...
+ * key = step * maxNumOfSteps, represents [step * maxNumOfSteps, MAX)
+ * </pre>
*/
public HeatMap readHeatMap(MetricsCondition condition, Duration duration) throws IOException {
return getMetricsQueryService().readHeatMap(condition, duration);
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
index 84b6790..278e19d 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
@@ -23,10 +23,8 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.skywalking.oap.server.core.analysis.DownSampling;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.analysis.metrics.HistogramMetrics;
-import org.apache.skywalking.oap.server.core.analysis.metrics.IntKeyLongValue;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.query.PointOfTime;
import org.apache.skywalking.oap.server.core.query.input.Duration;
@@ -36,7 +34,6 @@ import org.apache.skywalking.oap.server.core.query.type.HeatMap;
import org.apache.skywalking.oap.server.core.query.type.IntValues;
import org.apache.skywalking.oap.server.core.query.type.KVInt;
import org.apache.skywalking.oap.server.core.query.type.MetricsValues;
-import org.apache.skywalking.oap.server.core.query.type.Thermodynamic;
import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO;
import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
@@ -60,48 +57,6 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
}
@Override
- public Thermodynamic getThermodynamic(String indexName, DownSampling downsampling, List<String> ids,
- String valueCName) throws IOException {
- Thermodynamic thermodynamic = new Thermodynamic();
- List<List<Long>> thermodynamicValueMatrix = new ArrayList<>();
-
- SearchResponse response = getClient().ids(indexName, ids.toArray(new String[0]));
- Map<String, Map<String, Object>> idMap = toMap(response);
-
- int numOfSteps = 0;
- for (String id : ids) {
- Map<String, Object> source = idMap.get(id);
- if (source == null) {
- // add empty list to represent no data exist for this time bucket
- thermodynamicValueMatrix.add(new ArrayList<>());
- } else {
- int axisYStep = ((Number) source.get(HistogramMetrics.STEP)).intValue();
- thermodynamic.setAxisYStep(axisYStep);
- numOfSteps = ((Number) source.get(HistogramMetrics.NUM_OF_STEPS)).intValue() + 1;
-
- String value = (String) source.get(HistogramMetrics.DATASET);
- DataTable intKeyLongValues = new DataTable(5);
- intKeyLongValues.toObject(value);
-
- List<Long> axisYValues = new ArrayList<>();
- for (int i = 0; i < numOfSteps; i++) {
- axisYValues.add(0L);
- }
-
- for (IntKeyLongValue intKeyLongValue : intKeyLongValues.values()) {
- axisYValues.set(intKeyLongValue.getKey(), intKeyLongValue.getValue());
- }
-
- thermodynamicValueMatrix.add(axisYValues);
- }
- }
-
- thermodynamic.fromMatrixData(thermodynamicValueMatrix, numOfSteps);
-
- return thermodynamic;
- }
-
- @Override
public int readMetricsValue(final MetricsCondition condition,
final String valueColumnName,
final Duration duration) throws IOException {
@@ -221,7 +176,28 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
public HeatMap readHeatMap(final MetricsCondition condition,
final String valueColumnName,
final Duration duration) throws IOException {
- return null;
+ final List<PointOfTime> pointOfTimes = duration.assembleDurationPoints();
+ List<String> ids = new ArrayList<>(pointOfTimes.size());
+ pointOfTimes.forEach(pointOfTime -> {
+ ids.add(pointOfTime.id(condition.getEntity().buildId()));
+ });
+
+ SearchResponse response = getClient().ids(condition.getName(), ids.toArray(new String[0]));
+ Map<String, Map<String, Object>> idMap = toMap(response);
+
+ HeatMap heatMap = new HeatMap();
+
+ for (String id : ids) {
+ Map<String, Object> source = idMap.get(id);
+ if (source != null) {
+ String value = (String) source.get(HistogramMetrics.DATASET);
+ heatMap.buildColumn(id, value);
+ }
+ }
+
+ heatMap.fixMissingColumns(ids);
+
+ return heatMap;
}
protected void functionAggregation(Function function, TermsAggregationBuilder parentAggBuilder, String valueCName) {
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/MetricsQueryEs7DAO.java b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/MetricsQueryEs7DAO.java
index d1d9c59..977c3e0 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/MetricsQueryEs7DAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/MetricsQueryEs7DAO.java
@@ -19,12 +19,11 @@
package org.apache.skywalking.oap.server.storage.plugin.elasticsearch7.query;
import java.io.IOException;
-import org.apache.skywalking.oap.server.core.analysis.DownSampling;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
-import org.apache.skywalking.oap.server.core.query.type.IntValues;
-import org.apache.skywalking.oap.server.core.query.type.KVInt;
+import org.apache.skywalking.oap.server.core.query.input.Duration;
+import org.apache.skywalking.oap.server.core.query.input.MetricsCondition;
import org.apache.skywalking.oap.server.core.query.sql.Function;
-import org.apache.skywalking.oap.server.core.query.sql.Where;
+import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.MetricsQueryEsDAO;
import org.elasticsearch.action.search.SearchResponse;
@@ -35,6 +34,11 @@ import org.elasticsearch.search.aggregations.metrics.Avg;
import org.elasticsearch.search.aggregations.metrics.Sum;
import org.elasticsearch.search.builder.SearchSourceBuilder;
+/**
+ * Duplicate logic of {@link MetricsQueryEsDAO}, but for making compatible in ElasticSearch 7.
+ *
+ * {@link Avg} and {@link Sum} have different package in the es7.
+ */
public class MetricsQueryEs7DAO extends MetricsQueryEsDAO {
public MetricsQueryEs7DAO(ElasticSearchClient client) {
@@ -42,45 +46,37 @@ public class MetricsQueryEs7DAO extends MetricsQueryEsDAO {
}
@Override
- public IntValues getValues(String indexName, DownSampling downsampling, long startTB, long endTB, Where where,
- String valueCName, Function function) throws IOException {
+ public int readMetricsValue(final MetricsCondition condition,
+ final String valueColumnName,
+ final Duration duration) throws IOException {
SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
- buildQuery(sourceBuilder, where, startTB, endTB);
+ buildQuery(sourceBuilder, condition, duration);
TermsAggregationBuilder entityIdAggregation = AggregationBuilders.terms(Metrics.ENTITY_ID)
.field(Metrics.ENTITY_ID)
- .size(1000);
- functionAggregation(function, entityIdAggregation, valueCName);
+ .size(1);
+ final Function function = ValueColumnMetadata.INSTANCE.getValueFunction(condition.getName());
+ functionAggregation(function, entityIdAggregation, valueColumnName);
sourceBuilder.aggregation(entityIdAggregation);
- SearchResponse response = getClient().search(indexName, sourceBuilder);
+ SearchResponse response = getClient().search(condition.getName(), sourceBuilder);
- IntValues intValues = new IntValues();
Terms idTerms = response.getAggregations().get(Metrics.ENTITY_ID);
for (Terms.Bucket idBucket : idTerms.getBuckets()) {
- long value;
switch (function) {
case Sum:
- Sum sum = idBucket.getAggregations().get(valueCName);
- value = (long) sum.getValue();
- break;
+ Sum sum = idBucket.getAggregations().get(valueColumnName);
+ return (int) sum.getValue();
case Avg:
- Avg avg = idBucket.getAggregations().get(valueCName);
- value = (long) avg.getValue();
- break;
+ Avg avg = idBucket.getAggregations().get(valueColumnName);
+ return (int) avg.getValue();
default:
- avg = idBucket.getAggregations().get(valueCName);
- value = (long) avg.getValue();
- break;
+ avg = idBucket.getAggregations().get(valueColumnName);
+ return (int) avg.getValue();
}
-
- KVInt kvInt = new KVInt();
- kvInt.setId(idBucket.getKeyAsString());
- kvInt.setValue(value);
- intValues.addKVInt(kvInt);
}
- return intValues;
+ return ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
}
protected void functionAggregation(Function function, TermsAggregationBuilder parentAggBuilder, String valueCName) {