You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by lu...@apache.org on 2022/12/19 03:02:26 UTC
[skywalking] 01/01: refactor Topo query for BanyanDB
This is an automated email from the ASF dual-hosted git repository.
lujiajing pushed a commit to branch optimize-topology-query
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 9220ef96a6e3f7f184cfe92defd63cba1387f002
Author: Megrez Lu <lu...@gmail.com>
AuthorDate: Mon Dec 19 11:02:11 2022 +0800
refactor Topo query for BanyanDB
---
oap-server-bom/pom.xml | 2 +-
.../banyandb/measure/BanyanDBTopologyQueryDAO.java | 228 ++++++++++-----------
.../banyandb/stream/AbstractBanyanDBDAO.java | 8 +
3 files changed, 116 insertions(+), 122 deletions(-)
diff --git a/oap-server-bom/pom.xml b/oap-server-bom/pom.xml
index bf42981dca..30e7fa2bb4 100644
--- a/oap-server-bom/pom.xml
+++ b/oap-server-bom/pom.xml
@@ -73,7 +73,7 @@
<awaitility.version>3.0.0</awaitility.version>
<httpcore.version>4.4.13</httpcore.version>
<commons-compress.version>1.21</commons-compress.version>
- <banyandb-java-client.version>0.2.1</banyandb-java-client.version>
+ <banyandb-java-client.version>0.3.0-SNAPSHOT</banyandb-java-client.version>
<kafka-clients.version>2.8.1</kafka-clients.version>
<spring-kafka-test.version>2.4.6.RELEASE</spring-kafka-test.version>
</properties>
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java
index cc6f313b7a..928d0dcac9 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java
@@ -19,12 +19,17 @@
package org.apache.skywalking.oap.server.storage.plugin.banyandb.measure;
import com.google.common.collect.ImmutableSet;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.skywalking.banyandb.v1.client.AbstractCriteria;
import org.apache.skywalking.banyandb.v1.client.MeasureQuery;
import org.apache.skywalking.banyandb.v1.client.MeasureQueryResponse;
import org.apache.skywalking.banyandb.v1.client.TimestampRange;
@@ -62,9 +67,9 @@ public class BanyanDBTopologyQueryDAO extends AbstractBanyanDBDAO implements ITo
throw new UnexpectedException("Service id is empty");
}
- List<QueryBuilder<MeasureQuery>> queryBuilderList = buildServiceRelationsQueries(serviceIds);
+ QueryBuilder<MeasureQuery> queryBuilder = buildServiceRelationsQuery(serviceIds);
- return queryServiceRelation(duration, queryBuilderList, DetectPoint.SERVER);
+ return queryServiceRelation(duration, queryBuilder, DetectPoint.SERVER);
}
@Override
@@ -74,43 +79,34 @@ public class BanyanDBTopologyQueryDAO extends AbstractBanyanDBDAO implements ITo
throw new UnexpectedException("Service id is empty");
}
- List<QueryBuilder<MeasureQuery>> queryBuilderList = buildServiceRelationsQueries(serviceIds);
+ QueryBuilder<MeasureQuery> queryBuilder = buildServiceRelationsQuery(serviceIds);
- return queryServiceRelation(duration, queryBuilderList, DetectPoint.CLIENT);
+ return queryServiceRelation(duration, queryBuilder, DetectPoint.CLIENT);
}
@Override
public List<Call.CallDetail> loadServiceRelationsDetectedAtServerSide(Duration duration) throws IOException {
- return queryServiceRelation(duration, Collections.singletonList(emptyMeasureQuery()), DetectPoint.SERVER);
+ return queryServiceRelation(duration, emptyMeasureQuery(), DetectPoint.SERVER);
}
@Override
public List<Call.CallDetail> loadServiceRelationDetectedAtClientSide(Duration duration) throws IOException {
- return queryServiceRelation(duration, Collections.singletonList(emptyMeasureQuery()), DetectPoint.CLIENT);
+ return queryServiceRelation(duration, emptyMeasureQuery(), DetectPoint.CLIENT);
}
- private List<QueryBuilder<MeasureQuery>> buildServiceRelationsQueries(List<String> serviceIds) {
- List<QueryBuilder<MeasureQuery>> queryBuilderList = new ArrayList<>(serviceIds.size());
- for (final String serviceId : serviceIds) {
- queryBuilderList.add(new QueryBuilder<MeasureQuery>() {
- @Override
- protected void apply(MeasureQuery query) {
- query.and(eq(ServiceRelationServerSideMetrics.SOURCE_SERVICE_ID, serviceId));
- }
- });
-
- queryBuilderList.add(new QueryBuilder<MeasureQuery>() {
- @Override
- protected void apply(MeasureQuery query) {
- query.and(eq(ServiceRelationServerSideMetrics.DEST_SERVICE_ID, serviceId));
- }
- });
- }
- return queryBuilderList;
+ private QueryBuilder<MeasureQuery> buildServiceRelationsQuery(List<String> serviceIds) {
+ return new QueryBuilder<MeasureQuery>() {
+ @Override
+ protected void apply(MeasureQuery query) {
+ query.or(in(ServiceRelationServerSideMetrics.SOURCE_SERVICE_ID, serviceIds))
+ .or(in(ServiceRelationServerSideMetrics.DEST_SERVICE_ID, serviceIds));
+ query.groupBy(Sets.newHashSet(Metrics.ENTITY_ID, ServiceRelationServerSideMetrics.COMPONENT_IDS));
+ }
+ };
}
List<Call.CallDetail> queryServiceRelation(Duration duration,
- List<QueryBuilder<MeasureQuery>> queryBuilderList,
+ QueryBuilder<MeasureQuery> queryBuilder,
DetectPoint detectPoint) throws IOException {
long startTB = 0;
long endTB = 0;
@@ -123,27 +119,25 @@ public class BanyanDBTopologyQueryDAO extends AbstractBanyanDBDAO implements ITo
timestampRange = new TimestampRange(TimeBucket.getTimestamp(startTB), TimeBucket.getTimestamp(endTB));
}
final String modelName = detectPoint == DetectPoint.SERVER ? ServiceRelationServerSideMetrics.INDEX_NAME :
- ServiceRelationClientSideMetrics.INDEX_NAME;
+ ServiceRelationClientSideMetrics.INDEX_NAME;
final Map<String, Call.CallDetail> callMap = new HashMap<>();
- for (final QueryBuilder<MeasureQuery> q : queryBuilderList) {
- MeasureQueryResponse resp = query(modelName,
- ImmutableSet.of(
- ServiceRelationClientSideMetrics.COMPONENT_IDS,
- Metrics.ENTITY_ID
- ),
- Collections.emptySet(), timestampRange, q
- );
- if (resp.size() == 0) {
- continue;
- }
- final Call.CallDetail call = new Call.CallDetail();
- final String entityId = resp.getDataPoints().get(0).getTagValue(Metrics.ENTITY_ID);
- final IntList componentIds = new IntList(
+ MeasureQueryResponse resp = query(modelName,
+ ImmutableSet.of(
+ ServiceRelationClientSideMetrics.COMPONENT_IDS,
+ Metrics.ENTITY_ID
+ ),
+ Collections.emptySet(), timestampRange, queryBuilder
+ );
+ if (resp.size() == 0) {
+ return Collections.emptyList();
+ }
+ final Call.CallDetail call = new Call.CallDetail();
+ final String entityId = resp.getDataPoints().get(0).getTagValue(Metrics.ENTITY_ID);
+ final IntList componentIds = new IntList(
resp.getDataPoints().get(0).getTagValue(ServiceRelationClientSideMetrics.COMPONENT_IDS));
- for (int i = 0; i < componentIds.size(); i++) {
- call.buildFromServiceRelation(entityId, componentIds.get(i), detectPoint);
- callMap.putIfAbsent(entityId, call);
- }
+ for (int i = 0; i < componentIds.size(); i++) {
+ call.buildFromServiceRelation(entityId, componentIds.get(i), detectPoint);
+ callMap.putIfAbsent(entityId, call);
}
return new ArrayList<>(callMap.values());
}
@@ -152,43 +146,47 @@ public class BanyanDBTopologyQueryDAO extends AbstractBanyanDBDAO implements ITo
public List<Call.CallDetail> loadInstanceRelationDetectedAtServerSide(String clientServiceId,
String serverServiceId,
Duration duration) throws IOException {
- List<QueryBuilder<MeasureQuery>> queryBuilderList = buildInstanceRelationsQueries(
- clientServiceId, serverServiceId);
- return queryInstanceRelation(duration, queryBuilderList, DetectPoint.SERVER);
+ QueryBuilder<MeasureQuery> queryBuilder = buildInstanceRelationsQuery(
+ clientServiceId, serverServiceId);
+ return queryInstanceRelation(duration, queryBuilder, DetectPoint.SERVER);
}
@Override
public List<Call.CallDetail> loadInstanceRelationDetectedAtClientSide(String clientServiceId,
String serverServiceId,
Duration duration) throws IOException {
- List<QueryBuilder<MeasureQuery>> queryBuilderList = buildInstanceRelationsQueries(
- clientServiceId, serverServiceId);
- return queryInstanceRelation(duration, queryBuilderList, DetectPoint.CLIENT);
+ QueryBuilder<MeasureQuery> queryBuilder = buildInstanceRelationsQuery(
+ clientServiceId, serverServiceId);
+ return queryInstanceRelation(duration, queryBuilder, DetectPoint.CLIENT);
}
- private List<QueryBuilder<MeasureQuery>> buildInstanceRelationsQueries(String clientServiceId,
- String serverServiceId) {
- List<QueryBuilder<MeasureQuery>> queryBuilderList = new ArrayList<>(2);
- queryBuilderList.add(new QueryBuilder<MeasureQuery>() {
+ private QueryBuilder<MeasureQuery> buildInstanceRelationsQuery(String clientServiceId,
+ String serverServiceId) {
+ return new QueryBuilder<MeasureQuery>() {
@Override
protected void apply(MeasureQuery query) {
- query.and(eq(ServiceInstanceRelationServerSideMetrics.SOURCE_SERVICE_ID, clientServiceId))
- .and(eq(ServiceInstanceRelationServerSideMetrics.DEST_SERVICE_ID, serverServiceId));
- }
- });
+ List<AbstractCriteria> instanceRelationsQueryConditions = new ArrayList<>(2);
- queryBuilderList.add(new QueryBuilder<MeasureQuery>() {
- @Override
- protected void apply(MeasureQuery query) {
- query.and(eq(ServiceInstanceRelationServerSideMetrics.DEST_SERVICE_ID, clientServiceId))
- .and(eq(ServiceInstanceRelationServerSideMetrics.SOURCE_SERVICE_ID, serverServiceId));
+ instanceRelationsQueryConditions.add(
+ // source_service_id = clientServiceId AND dest_service_id = serverServiceId
+ and(Lists.newArrayList(
+ eq(ServiceInstanceRelationServerSideMetrics.SOURCE_SERVICE_ID, clientServiceId),
+ eq(ServiceInstanceRelationServerSideMetrics.DEST_SERVICE_ID, serverServiceId))
+ ));
+ instanceRelationsQueryConditions.add(
+ // source_service_id = clientServiceId AND dest_service_id = serverServiceId
+ and(Lists.newArrayList(
+ eq(ServiceInstanceRelationServerSideMetrics.DEST_SERVICE_ID, clientServiceId),
+ eq(ServiceInstanceRelationServerSideMetrics.SOURCE_SERVICE_ID, serverServiceId)
+ ))
+ );
+ query.criteria(or(instanceRelationsQueryConditions));
}
- });
- return queryBuilderList;
+ };
}
List<Call.CallDetail> queryInstanceRelation(Duration duration,
- List<QueryBuilder<MeasureQuery>> queryBuilderList,
+ QueryBuilder<MeasureQuery> queryBuilder,
DetectPoint detectPoint) throws IOException {
long startTB = 0;
long endTB = 0;
@@ -201,30 +199,28 @@ public class BanyanDBTopologyQueryDAO extends AbstractBanyanDBDAO implements ITo
timestampRange = new TimestampRange(TimeBucket.getTimestamp(startTB), TimeBucket.getTimestamp(endTB));
}
final String modelName = detectPoint == DetectPoint.SERVER ? ServiceInstanceRelationServerSideMetrics.INDEX_NAME :
- ServiceInstanceRelationClientSideMetrics.INDEX_NAME;
+ ServiceInstanceRelationClientSideMetrics.INDEX_NAME;
final Map<String, Call.CallDetail> callMap = new HashMap<>();
- for (final QueryBuilder<MeasureQuery> q : queryBuilderList) {
- MeasureQueryResponse resp = query(modelName,
- ImmutableSet.of(
- Metrics.ENTITY_ID
- ),
- Collections.emptySet(), timestampRange, q
- );
- if (resp.size() == 0) {
- continue;
- }
- final Call.CallDetail call = new Call.CallDetail();
- final String entityId = resp.getDataPoints().get(0).getTagValue(Metrics.ENTITY_ID);
- call.buildFromInstanceRelation(entityId, detectPoint);
- callMap.putIfAbsent(entityId, call);
+ MeasureQueryResponse resp = query(modelName,
+ ImmutableSet.of(
+ Metrics.ENTITY_ID
+ ),
+ Collections.emptySet(), timestampRange, queryBuilder
+ );
+ if (resp.size() == 0) {
+ return Collections.emptyList();
}
+ final Call.CallDetail call = new Call.CallDetail();
+ final String entityId = resp.getDataPoints().get(0).getTagValue(Metrics.ENTITY_ID);
+ call.buildFromInstanceRelation(entityId, detectPoint);
+ callMap.putIfAbsent(entityId, call);
return new ArrayList<>(callMap.values());
}
@Override
public List<Call.CallDetail> loadEndpointRelation(Duration duration, String destEndpointId) throws IOException {
- List<QueryBuilder<MeasureQuery>> queryBuilderList = buildEndpointRelationsQueries(destEndpointId);
- return queryEndpointRelation(duration, queryBuilderList, DetectPoint.SERVER);
+ QueryBuilder<MeasureQuery> queryBuilder = buildEndpointRelationsQueries(destEndpointId);
+ return queryEndpointRelation(duration, queryBuilder, DetectPoint.SERVER);
}
@Override
@@ -239,26 +235,18 @@ public class BanyanDBTopologyQueryDAO extends AbstractBanyanDBDAO implements ITo
return queryProcessRelation(duration, serviceInstanceId, DetectPoint.SERVER);
}
- private List<QueryBuilder<MeasureQuery>> buildEndpointRelationsQueries(String destEndpointId) {
- List<QueryBuilder<MeasureQuery>> queryBuilderList = new ArrayList<>(2);
- queryBuilderList.add(new QueryBuilder<MeasureQuery>() {
+ private QueryBuilder<MeasureQuery> buildEndpointRelationsQueries(String destEndpointId) {
+ return new QueryBuilder<MeasureQuery>() {
@Override
protected void apply(MeasureQuery query) {
- query.and(eq(EndpointRelationServerSideMetrics.SOURCE_ENDPOINT, destEndpointId));
+ query.or(eq(EndpointRelationServerSideMetrics.SOURCE_ENDPOINT, destEndpointId))
+ .or(eq(EndpointRelationServerSideMetrics.DEST_ENDPOINT, destEndpointId));
}
- });
-
- queryBuilderList.add(new QueryBuilder<MeasureQuery>() {
- @Override
- protected void apply(MeasureQuery query) {
- query.and(eq(EndpointRelationServerSideMetrics.DEST_ENDPOINT, destEndpointId));
- }
- });
- return queryBuilderList;
+ };
}
List<Call.CallDetail> queryEndpointRelation(Duration duration,
- List<QueryBuilder<MeasureQuery>> queryBuilderList,
+ QueryBuilder<MeasureQuery> queryBuilder,
DetectPoint detectPoint) throws IOException {
long startTB = 0;
long endTB = 0;
@@ -271,21 +259,19 @@ public class BanyanDBTopologyQueryDAO extends AbstractBanyanDBDAO implements ITo
timestampRange = new TimestampRange(TimeBucket.getTimestamp(startTB), TimeBucket.getTimestamp(endTB));
}
final Map<String, Call.CallDetail> callMap = new HashMap<>();
- for (final QueryBuilder<MeasureQuery> q : queryBuilderList) {
- MeasureQueryResponse resp = query(EndpointRelationServerSideMetrics.INDEX_NAME,
- ImmutableSet.of(
- Metrics.ENTITY_ID
- ),
- Collections.emptySet(), timestampRange, q
- );
- if (resp.size() == 0) {
- continue;
- }
- final Call.CallDetail call = new Call.CallDetail();
- final String entityId = resp.getDataPoints().get(0).getTagValue(Metrics.ENTITY_ID);
- call.buildFromEndpointRelation(entityId, detectPoint);
- callMap.putIfAbsent(entityId, call);
+ MeasureQueryResponse resp = query(EndpointRelationServerSideMetrics.INDEX_NAME,
+ ImmutableSet.of(
+ Metrics.ENTITY_ID
+ ),
+ Collections.emptySet(), timestampRange, queryBuilder
+ );
+ if (resp.size() == 0) {
+ return Collections.emptyList();
}
+ final Call.CallDetail call = new Call.CallDetail();
+ final String entityId = resp.getDataPoints().get(0).getTagValue(Metrics.ENTITY_ID);
+ call.buildFromEndpointRelation(entityId, detectPoint);
+ callMap.putIfAbsent(entityId, call);
return new ArrayList<>(callMap.values());
}
@@ -303,23 +289,23 @@ public class BanyanDBTopologyQueryDAO extends AbstractBanyanDBDAO implements ITo
timestampRange = new TimestampRange(TimeBucket.getTimestamp(startTB), TimeBucket.getTimestamp(endTB));
}
final String modelName = detectPoint == DetectPoint.SERVER ? ProcessRelationServerSideMetrics.INDEX_NAME :
- ProcessRelationClientSideMetrics.INDEX_NAME;
+ ProcessRelationClientSideMetrics.INDEX_NAME;
final Map<String, Call.CallDetail> callMap = new HashMap<>();
MeasureQueryResponse resp = query(modelName,
- ImmutableSet.of(
- Metrics.ENTITY_ID, ProcessRelationClientSideMetrics.COMPONENT_ID),
- Collections.emptySet(), timestampRange, new QueryBuilder<MeasureQuery>() {
- @Override
- protected void apply(MeasureQuery query) {
- query.and(eq(ProcessRelationServerSideMetrics.SERVICE_INSTANCE_ID, serviceInstanceId));
+ ImmutableSet.of(
+ Metrics.ENTITY_ID, ProcessRelationClientSideMetrics.COMPONENT_ID),
+ Collections.emptySet(), timestampRange, new QueryBuilder<MeasureQuery>() {
+ @Override
+ protected void apply(MeasureQuery query) {
+ query.and(eq(ProcessRelationServerSideMetrics.SERVICE_INSTANCE_ID, serviceInstanceId));
+ }
}
- }
);
final Call.CallDetail call = new Call.CallDetail();
final String entityId = resp.getDataPoints().get(0).getTagValue(Metrics.ENTITY_ID);
final int componentId = ((Number) resp.getDataPoints()
- .get(0)
- .getTagValue(ProcessRelationClientSideMetrics.COMPONENT_ID)).intValue();
+ .get(0)
+ .getTagValue(ProcessRelationClientSideMetrics.COMPONENT_ID)).intValue();
call.buildProcessRelation(entityId, componentId, detectPoint);
callMap.putIfAbsent(entityId, call);
return new ArrayList<>(callMap.values());
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java
index 4978b3f64a..62f3facf0c 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java
@@ -134,6 +134,14 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO<BanyanDBStorageCli
return PairQueryCondition.StringQueryCondition.eq(name, value);
}
+ protected PairQueryCondition<List<String>> in(String name, List<String> values) {
+ return PairQueryCondition.StringArrayQueryCondition.in(name, values);
+ }
+
+ protected PairQueryCondition<List<String>> notIn(String name, List<String> values) {
+ return PairQueryCondition.StringArrayQueryCondition.in(name, values);
+ }
+
protected PairQueryCondition<Long> ne(String name, long value) {
return PairQueryCondition.LongQueryCondition.ne(name, value);
}