You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2020/07/29 14:43:17 UTC
[skywalking] branch query-labeled created (now e358683)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a change to branch query-labeled
in repository https://gitbox.apache.org/repos/asf/skywalking.git.
at e358683 Query all of labels once selected labels condition is absent.
This branch includes the following new commits:
new e358683 Query all of labels once selected labels condition is absent.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[skywalking] 01/01: Query all of labels once selected labels
condition is absent.
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch query-labeled
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit e35868394e1d2caee27051e1a629115f6c808734
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed Jul 29 22:42:06 2020 +0800
Query all of labels once selected labels condition is absent.
---
.../server/core/analysis/metrics/DataTable.java | 5 +
.../core/storage/query/IMetricsQueryDAO.java | 51 +++++++++
.../core/storage/query/MetricsQueryUtilTest.java | 114 +++++++++++++++++++++
.../elasticsearch/query/MetricsQueryEsDAO.java | 41 ++------
.../plugin/influxdb/query/MetricsQuery.java | 30 +-----
.../plugin/jdbc/h2/dao/H2MetricsQueryDAO.java | 30 +-----
6 files changed, 182 insertions(+), 89 deletions(-)
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTable.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTable.java
index a104ee6..2fca2da 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTable.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTable.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
+import java.util.Set;
import java.util.stream.Collectors;
import lombok.EqualsAndHashCode;
import lombok.ToString;
@@ -94,6 +95,10 @@ public class DataTable implements StorageDataComplexObject<DataTable> {
return values;
}
+ public Set<String> keys() {
+ return data.keySet();
+ }
+
public boolean hasData() {
return !data.isEmpty();
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java
index 79be899..803c6bb 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java
@@ -18,8 +18,14 @@
package org.apache.skywalking.oap.server.core.storage.query;
+import io.vavr.Tuple;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.query.input.Duration;
import org.apache.skywalking.oap.server.core.query.input.MetricsCondition;
import org.apache.skywalking.oap.server.core.query.type.HeatMap;
@@ -27,6 +33,11 @@ import org.apache.skywalking.oap.server.core.query.type.IntValues;
import org.apache.skywalking.oap.server.core.query.type.KVInt;
import org.apache.skywalking.oap.server.core.query.type.MetricsValues;
import org.apache.skywalking.oap.server.core.storage.DAO;
+import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
+
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.mapping;
+import static java.util.stream.Collectors.toList;
/**
* Query metrics values in different ways.
@@ -76,5 +87,45 @@ public interface IMetricsQueryDAO extends DAO {
}
return origin;
}
+
+ /**
+ * Compose the multiple metric result based on conditions.
+ */
+ public static List<MetricsValues> composeLabelValue(final MetricsCondition condition,
+ final List<String> labels,
+ final List<String> ids,
+ final Map<String, DataTable> idMap) {
+ List<String> allLabels;
+ if (Objects.isNull(labels) || labels.size() < 1) {
+ allLabels = idMap.values().stream()
+ .flatMap(dataTable -> dataTable.keys().stream())
+ .distinct().collect(Collectors.toList());
+ } else {
+ allLabels = labels;
+ }
+ final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
+ return allLabels.stream()
+ .flatMap(label -> ids.stream()
+ .map(id -> Tuple.of(
+ label,
+ id,
+ Optional.ofNullable(idMap.getOrDefault(id, new DataTable()).get(label)).orElse(0L))))
+ .collect(groupingBy(t -> t._1, mapping(t -> {
+ KVInt kv = new KVInt();
+ kv.setId(t._2);
+ kv.setValue(t._3);
+ return kv;
+ }, toList())))
+ .entrySet().stream()
+ .map(entry -> {
+ MetricsValues labelValue = new MetricsValues();
+ labelValue.setLabel(entry.getKey());
+ IntValues values = new IntValues();
+ entry.getValue().forEach(values::addKVInt);
+ labelValue.setValues(sortValues(values, ids, defaultValue));
+ return labelValue;
+ })
+ .collect(toList());
+ }
}
}
diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/query/MetricsQueryUtilTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/query/MetricsQueryUtilTest.java
new file mode 100644
index 0000000..3283e1e
--- /dev/null
+++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/query/MetricsQueryUtilTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.storage.query;
+
+import com.google.gson.Gson;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import lombok.RequiredArgsConstructor;
+import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
+import org.apache.skywalking.oap.server.core.query.input.MetricsCondition;
+import org.apache.skywalking.oap.server.core.query.sql.Function;
+import org.apache.skywalking.oap.server.core.query.type.MetricsValues;
+import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static com.google.common.collect.ImmutableMap.of;
+import static java.util.Arrays.asList;
+import static org.apache.skywalking.oap.server.core.storage.annotation.Column.ValueDataType.LABELED_VALUE;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+
+@RunWith(Parameterized.class)
+@RequiredArgsConstructor
+public class MetricsQueryUtilTest {
+
+ private static final String MODULE_NAME = "meter-test";
+
+ private final List<String> queryConditionLabels;
+
+ private final List<String> datePoints;
+
+ private final Map<String, DataTable> valueColumnData;
+
+ private final String expectedResult;
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ {
+ asList("200", "400"),
+ asList("202007291425", "202007291426"),
+ of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
+ "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," +
+ "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}]"
+ },
+ {
+ Collections.emptyList(),
+ asList("202007291425", "202007291426"),
+ of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
+ "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," +
+ "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}]"
+ },
+ {
+ Collections.singletonList("200"),
+ asList("202007291425", "202007291426"),
+ of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
+ "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}]"
+ },
+ {
+ asList("200", "400", "500"),
+ asList("202007291425", "202007291426"),
+ of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
+ "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," +
+ "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}," +
+ "{\"label\":\"500\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":0},{\"id\":\"202007291426\",\"value\":0}]}}]"
+ },
+ {
+ asList("200", "400"),
+ asList("202007291425", "202007291426"),
+ of("202007291425", new DataTable("200,1|400,2")),
+ "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":0}]}}," +
+ "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":0}]}}]"
+ },
+ });
+ }
+
+ @Before
+ public void setup() {
+ ValueColumnMetadata.INSTANCE.putIfAbsent(
+ MODULE_NAME, "value", LABELED_VALUE, Function.None, 0
+ );
+ }
+
+ @Test
+ public void testComposeLabelValue() {
+ MetricsCondition condition = new MetricsCondition();
+ condition.setName(MODULE_NAME);
+ List<MetricsValues> result = IMetricsQueryDAO.Util.composeLabelValue(condition, queryConditionLabels, datePoints, valueColumnData);
+ assertThat(new Gson().toJson(result), is(expectedResult));
+ }
+
+}
\ No newline at end of file
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
index c49ca9a..5145de5 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java
@@ -21,6 +21,7 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
@@ -138,42 +139,12 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
});
SearchResponse response = getClient().ids(condition.getName(), ids.toArray(new String[0]));
- Map<String, Map<String, Object>> idMap = toMap(response);
-
- Map<String, MetricsValues> labeledValues = new HashMap<>(labels.size());
- labels.forEach(label -> {
- MetricsValues labelValue = new MetricsValues();
- labelValue.setLabel(label);
-
- labeledValues.put(label, labelValue);
- });
-
- final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
- for (String id : ids) {
- if (idMap.containsKey(id)) {
- Map<String, Object> source = idMap.get(id);
- DataTable multipleValues = new DataTable((String) source.getOrDefault(valueColumnName, ""));
-
- labels.forEach(label -> {
- final IntValues values = labeledValues.get(label).getValues();
- Long data = multipleValues.get(label);
- if (data == null) {
- data = (long) defaultValue;
- }
- KVInt kv = new KVInt();
- kv.setId(id);
- kv.setValue(data);
- values.addKVInt(kv);
- });
- }
-
+ Map<String, DataTable> idMap = new LinkedHashMap<>();
+ SearchHit[] hits = response.getHits().getHits();
+ for (SearchHit hit : hits) {
+ idMap.put(hit.getId(), new DataTable((String) hit.getSourceAsMap().getOrDefault(valueColumnName, "")));
}
-
- return Util.sortValues(
- new ArrayList<>(labeledValues.values()),
- ids,
- defaultValue
- );
+ return Util.composeLabelValue(condition, labels, ids, idMap);
}
@Override
diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java
index dc8842a..f250d4f 100644
--- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java
+++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java
@@ -175,40 +175,16 @@ public class MetricsQuery implements IMetricsQueryDAO {
log.debug("SQL: {} result set: {}", query.getCommand(), series);
}
- Map<String, MetricsValues> labeledValues = new HashMap<>(labels.size());
- labels.forEach(label -> {
- MetricsValues labelValue = new MetricsValues();
- labelValue.setLabel(label);
-
- labeledValues.put(label, labelValue);
- });
-
- final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
+ Map<String, DataTable> idMap = new HashMap<>();
if (!CollectionUtils.isEmpty(series)) {
series.get(0).getValues().forEach(values -> {
final String id = (String) values.get(1);
DataTable multipleValues = new DataTable(5);
multipleValues.toObject((String) values.get(2));
-
- labels.forEach(label -> {
- Long data = multipleValues.get(label);
- if (data == null) {
- data = (long) defaultValue;
- }
- final IntValues intValues = labeledValues.get(label).getValues();
- KVInt kv = new KVInt();
- kv.setId(id);
- kv.setValue(data);
- intValues.addKVInt(kv);
- });
+ idMap.put(id, multipleValues);
});
}
-
- return Util.sortValues(
- new ArrayList<>(labeledValues.values()),
- ids,
- defaultValue
- );
+ return Util.composeLabelValue(condition, labels, ids, idMap);
}
@Override
diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java
index 84d0ca0..f2b0f32 100644
--- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java
@@ -162,16 +162,7 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO
}
sql.append(")");
- Map<String, MetricsValues> labeledValues = new HashMap<>(labels.size());
- labels.forEach(label -> {
- MetricsValues labelValue = new MetricsValues();
- labelValue.setLabel(label);
-
- labeledValues.put(label, labelValue);
- });
-
- final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
-
+ Map<String, DataTable> idMap = new HashMap<>();
try (Connection connection = h2Client.getConnection()) {
try (ResultSet resultSet = h2Client.executeQuery(
connection, sql.toString(), parameters.toArray(new Object[0]))) {
@@ -181,28 +172,13 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO
DataTable multipleValues = new DataTable(5);
multipleValues.toObject(resultSet.getString(valueColumnName));
- labels.forEach(label -> {
- Long data = multipleValues.get(label);
- if (data == null) {
- data = (long) defaultValue;
- }
- final IntValues values = labeledValues.get(label).getValues();
- KVInt kv = new KVInt();
- kv.setId(id);
- kv.setValue(data);
- values.addKVInt(kv);
- });
+ idMap.put(id, multipleValues);
}
}
} catch (SQLException e) {
throw new IOException(e);
}
-
- return Util.sortValues(
- new ArrayList<>(labeledValues.values()),
- ids,
- defaultValue
- );
+ return Util.composeLabelValue(condition, labels, ids, idMap);
}
@Override