You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2020/07/29 14:43:18 UTC
[skywalking] 01/01: Query all of labels once selected labels
condition is absent.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch query-labeled
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit e35868394e1d2caee27051e1a629115f6c808734
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed Jul 29 22:42:06 2020 +0800
Query all of labels once selected labels condition is absent.
---
.../server/core/analysis/metrics/DataTable.java | 5 +
.../core/storage/query/IMetricsQueryDAO.java | 51 +++++++++
.../core/storage/query/MetricsQueryUtilTest.java | 114 +++++++++++++++++++++
.../elasticsearch/query/MetricsQueryEsDAO.java | 41 ++------
.../plugin/influxdb/query/MetricsQuery.java | 30 +-----
.../plugin/jdbc/h2/dao/H2MetricsQueryDAO.java | 30 +-----
6 files changed, 182 insertions(+), 89 deletions(-)
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTable.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTable.java
index a104ee6..2fca2da 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTable.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTable.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
+import java.util.Set;
import java.util.stream.Collectors;
import lombok.EqualsAndHashCode;
import lombok.ToString;
@@ -94,6 +95,10 @@ public class DataTable implements StorageDataComplexObject<DataTable> {
return values;
}
+ public Set<String> keys() {
+ return data.keySet();
+ }
+
public boolean hasData() {
return !data.isEmpty();
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java
index 79be899..803c6bb 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java
@@ -18,8 +18,14 @@
package org.apache.skywalking.oap.server.core.storage.query;
+import io.vavr.Tuple;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.query.input.Duration;
import org.apache.skywalking.oap.server.core.query.input.MetricsCondition;
import org.apache.skywalking.oap.server.core.query.type.HeatMap;
@@ -27,6 +33,11 @@ import org.apache.skywalking.oap.server.core.query.type.IntValues;
import org.apache.skywalking.oap.server.core.query.type.KVInt;
import org.apache.skywalking.oap.server.core.query.type.MetricsValues;
import org.apache.skywalking.oap.server.core.storage.DAO;
+import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
+
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.mapping;
+import static java.util.stream.Collectors.toList;
/**
* Query metrics values in different ways.
@@ -76,5 +87,45 @@ public interface IMetricsQueryDAO extends DAO {
}
return origin;
}
+
+ /**
+ * Compose the multiple metric result based on conditions.
+ */
+ public static List<MetricsValues> composeLabelValue(final MetricsCondition condition,
+ final List<String> labels,
+ final List<String> ids,
+ final Map<String, DataTable> idMap) {
+ List<String> allLabels;
+ if (Objects.isNull(labels) || labels.size() < 1) {
+ allLabels = idMap.values().stream()
+ .flatMap(dataTable -> dataTable.keys().stream())
+ .distinct().collect(Collectors.toList());
+ } else {
+ allLabels = labels;
+ }
+ final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
+ return allLabels.stream()
+ .flatMap(label -> ids.stream()
+ .map(id -> Tuple.of(
+ label,
+ id,
+ Optional.ofNullable(idMap.getOrDefault(id, new DataTable()).get(label)).orElse(0L))))
+ .collect(groupingBy(t -> t._1, mapping(t -> {
+ KVInt kv = new KVInt();
+ kv.setId(t._2);
+ kv.setValue(t._3);
+ return kv;
+ }, toList())))
+ .entrySet().stream()
+ .map(entry -> {
+ MetricsValues labelValue = new MetricsValues();
+ labelValue.setLabel(entry.getKey());
+ IntValues values = new IntValues();
+ entry.getValue().forEach(values::addKVInt);
+ labelValue.setValues(sortValues(values, ids, defaultValue));
+ return labelValue;
+ })
+ .collect(toList());
+ }
}
}
diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/query/MetricsQueryUtilTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/query/MetricsQueryUtilTest.java
new file mode 100644
index 0000000..3283e1e
--- /dev/null
+++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/query/MetricsQueryUtilTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.storage.query;
+
+import com.google.gson.Gson;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import lombok.RequiredArgsConstructor;
+import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
+import org.apache.skywalking.oap.server.core.query.input.MetricsCondition;
+import org.apache.skywalking.oap.server.core.query.sql.Function;
+import org.apache.skywalking.oap.server.core.query.type.MetricsValues;
+import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static com.google.common.collect.ImmutableMap.of;
+import static java.util.Arrays.asList;
+import static org.apache.skywalking.oap.server.core.storage.annotation.Column.ValueDataType.LABELED_VALUE;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+
+@RunWith(Parameterized.class)
+@RequiredArgsConstructor
+public class MetricsQueryUtilTest {
+
+ private static final String MODULE_NAME = "meter-test";
+
+ private final List<String> queryConditionLabels;
+
+ private final List<String> datePoints;
+
+ private final Map<String, DataTable> valueColumnData;
+
+ private final String expectedResult;
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ {
+ asList("200", "400"),
+ asList("202007291425", "202007291426"),
+ of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
+ "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," +
+ "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}]"
+ },
+ {
+ Collections.emptyList(),
+ asList("202007291425", "202007291426"),
+ of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
+ "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," +
+ "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}]"
+ },
+ {
+ Collections.singletonList("200"),
+ asList("202007291425", "202007291426"),
+ of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
+ "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}]"
+ },
+ {
+ asList("200", "400", "500"),
+ asList("202007291425", "202007291426"),
+ of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
+ "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," +
+ "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}," +
+ "{\"label\":\"500\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":0},{\"id\":\"202007291426\",\"value\":0}]}}]"
+ },
+ {
+ asList("200", "400"),
+ asList("202007291425", "202007291426"),
+ of("202007291425", new DataTable("200,1|400,2")),
+ "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":0}]}}," +
+ "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":0}]}}]"
+ },
+ });
+ }
+
+ @Before
+ public void setup() {
+ ValueColumnMetadata.INSTANCE.putIfAbsent(
+ MODULE_NAME, "value", LABELED_VALUE, Function.None, 0
+ );
+ }
+
+ @Test
+ public void testComposeLabelValue() {
+ MetricsCondition condition = new MetricsCondition();
+ condition.setName(MODULE_NAME);
+ List<MetricsValues> result = IMetricsQueryDAO.Util.composeLabelValue(condition, queryConditionLabels, datePoints, valueColumnData);
+ assertThat(new Gson().toJson(result), is(expectedResult));
+ }
+
+}
\ No newline at end of file
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
index c49ca9a..5145de5 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
@@ -21,6 +21,7 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
@@ -138,42 +139,12 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
});
SearchResponse response = getClient().ids(condition.getName(), ids.toArray(new String[0]));
- Map<String, Map<String, Object>> idMap = toMap(response);
-
- Map<String, MetricsValues> labeledValues = new HashMap<>(labels.size());
- labels.forEach(label -> {
- MetricsValues labelValue = new MetricsValues();
- labelValue.setLabel(label);
-
- labeledValues.put(label, labelValue);
- });
-
- final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
- for (String id : ids) {
- if (idMap.containsKey(id)) {
- Map<String, Object> source = idMap.get(id);
- DataTable multipleValues = new DataTable((String) source.getOrDefault(valueColumnName, ""));
-
- labels.forEach(label -> {
- final IntValues values = labeledValues.get(label).getValues();
- Long data = multipleValues.get(label);
- if (data == null) {
- data = (long) defaultValue;
- }
- KVInt kv = new KVInt();
- kv.setId(id);
- kv.setValue(data);
- values.addKVInt(kv);
- });
- }
-
+ Map<String, DataTable> idMap = new LinkedHashMap<>();
+ SearchHit[] hits = response.getHits().getHits();
+ for (SearchHit hit : hits) {
+ idMap.put(hit.getId(), new DataTable((String) hit.getSourceAsMap().getOrDefault(valueColumnName, "")));
}
-
- return Util.sortValues(
- new ArrayList<>(labeledValues.values()),
- ids,
- defaultValue
- );
+ return Util.composeLabelValue(condition, labels, ids, idMap);
}
@Override
diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java
index dc8842a..f250d4f 100644
--- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java
+++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java
@@ -175,40 +175,16 @@ public class MetricsQuery implements IMetricsQueryDAO {
log.debug("SQL: {} result set: {}", query.getCommand(), series);
}
- Map<String, MetricsValues> labeledValues = new HashMap<>(labels.size());
- labels.forEach(label -> {
- MetricsValues labelValue = new MetricsValues();
- labelValue.setLabel(label);
-
- labeledValues.put(label, labelValue);
- });
-
- final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
+ Map<String, DataTable> idMap = new HashMap<>();
if (!CollectionUtils.isEmpty(series)) {
series.get(0).getValues().forEach(values -> {
final String id = (String) values.get(1);
DataTable multipleValues = new DataTable(5);
multipleValues.toObject((String) values.get(2));
-
- labels.forEach(label -> {
- Long data = multipleValues.get(label);
- if (data == null) {
- data = (long) defaultValue;
- }
- final IntValues intValues = labeledValues.get(label).getValues();
- KVInt kv = new KVInt();
- kv.setId(id);
- kv.setValue(data);
- intValues.addKVInt(kv);
- });
+ idMap.put(id, multipleValues);
});
}
-
- return Util.sortValues(
- new ArrayList<>(labeledValues.values()),
- ids,
- defaultValue
- );
+ return Util.composeLabelValue(condition, labels, ids, idMap);
}
@Override
diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java
index 84d0ca0..f2b0f32 100644
--- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java
@@ -162,16 +162,7 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO
}
sql.append(")");
- Map<String, MetricsValues> labeledValues = new HashMap<>(labels.size());
- labels.forEach(label -> {
- MetricsValues labelValue = new MetricsValues();
- labelValue.setLabel(label);
-
- labeledValues.put(label, labelValue);
- });
-
- final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
-
+ Map<String, DataTable> idMap = new HashMap<>();
try (Connection connection = h2Client.getConnection()) {
try (ResultSet resultSet = h2Client.executeQuery(
connection, sql.toString(), parameters.toArray(new Object[0]))) {
@@ -181,28 +172,13 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO
DataTable multipleValues = new DataTable(5);
multipleValues.toObject(resultSet.getString(valueColumnName));
- labels.forEach(label -> {
- Long data = multipleValues.get(label);
- if (data == null) {
- data = (long) defaultValue;
- }
- final IntValues values = labeledValues.get(label).getValues();
- KVInt kv = new KVInt();
- kv.setId(id);
- kv.setValue(data);
- values.addKVInt(kv);
- });
+ idMap.put(id, multipleValues);
}
}
} catch (SQLException e) {
throw new IOException(e);
}
-
- return Util.sortValues(
- new ArrayList<>(labeledValues.values()),
- ids,
- defaultValue
- );
+ return Util.composeLabelValue(condition, labels, ids, idMap);
}
@Override