You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2023/10/04 14:24:47 UTC

[skywalking] branch master updated: Update BanyanDB Java Client to 0.5.0 (#11370)

This is an automated email from the ASF dual-hosted git repository.

wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git


The following commit(s) were added to refs/heads/master by this push:
     new 338ebcaaa1 Update BanyanDB Java Client to 0.5.0 (#11370)
338ebcaaa1 is described below

commit 338ebcaaa105fb37672161c2e73745e0d0fe3143
Author: Jiajing LU <lu...@gmail.com>
AuthorDate: Wed Oct 4 09:24:41 2023 -0500

    Update BanyanDB Java Client to 0.5.0 (#11370)
---
 docs/en/changes/changes.md                         |  2 ++
 docs/en/setup/backend/configuration-vocabulary.md  |  3 +-
 docs/en/setup/backend/storages/banyandb.md         |  7 ++--
 oap-server-bom/pom.xml                             |  2 +-
 .../src/main/resources/application.yml             |  3 +-
 .../plugin/banyandb/BanyanDBStorageClient.java     |  4 +--
 .../plugin/banyandb/BanyanDBStorageConfig.java     | 14 ++++++--
 .../plugin/banyandb/BanyanDBStorageProvider.java   |  2 +-
 .../banyandb/measure/BanyanDBMetadataQueryDAO.java | 39 +++++++++++++---------
 test/e2e-v2/script/docker-compose/base-compose.yml |  2 +-
 test/e2e-v2/script/env                             |  2 +-
 11 files changed, 49 insertions(+), 31 deletions(-)

diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 5c0fe0f22c..6cb4d3ccfc 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -13,6 +13,8 @@
   including `ui_template,ui_menu,continuous_profiling_policy`.
 * Add a release mechanism for alarm windows when it is expired in case of OOM.
 * Fix Zipkin trace receiver response: make the HTTP status code from `200` to `202`.
+* Update BanyanDB Java Client to 0.5.0.
+* Fix getInstances query in the BanyanDB Metadata DAO.
 
 #### UI
 
diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md
index c125103eb7..961dde11e2 100644
--- a/docs/en/setup/backend/configuration-vocabulary.md
+++ b/docs/en/setup/backend/configuration-vocabulary.md
@@ -139,8 +139,7 @@ The Configuration Vocabulary lists all available configurations provided by `app
 | -                       | -             | maxSizeOfBatchSql                                                                                                                                                        | The maximum size of batch size of SQL execution                                                                                                                                                                                                                                       [...]
 | -                       | -             | asyncBatchPersistentPoolSize                                                                                                                                             | async flush data into database thread size                                                                                                                                                                                                                                            [...]
 | -                       | banyandb      | -                                                                                                                                                                        | BanyanDB storage.                                                                                                                                                                                                                                                                     [...]
-| -                       | -             | host                                                                                                                                                                     | Host of the BanyanDB.                                                                                                                                                                                                                                                                 [...]
-| -                       | -             | port                                                                                                                                                                     | Port of the BanyanDB.                                                                                                                                                                                                                                                                 [...]
+| -                       | -             | targets                                                                                                                                                                  | Hosts with ports of the BanyanDB.                                                                                                                                                                                                                                                     [...]
 | -                       | -             | maxBulkSize                                                                                                                                                              | The maximum size of write entities in a single batch write call.                                                                                                                                                                                                                      [...]
 | -                       | -             | flushInterval                                                                                                                                                            | Period of flush interval. In the timeunit of seconds.                                                                                                                                                                                                                                 [...]
 | -                       | -             | metricsShardsNumber                                                                                                                                                      | Shards Number for measure/metrics.                                                                                                                                                                                                                                                    [...]
diff --git a/docs/en/setup/backend/storages/banyandb.md b/docs/en/setup/backend/storages/banyandb.md
index 40e368bf2d..c7271aef2f 100644
--- a/docs/en/setup/backend/storages/banyandb.md
+++ b/docs/en/setup/backend/storages/banyandb.md
@@ -3,11 +3,12 @@
 [BanyanDB](https://github.com/apache/skywalking-banyandb) is a dedicated storage implementation developed by the SkyWalking Team and the community.
 Activate BanyanDB as the storage, and set storage provider to **banyandb**.
 
+The OAP requires BanyanDB 0.5 server. As BanyanDB is still in the beta phase, we don't provide any compatibility besides the required version.
+
 ```yaml
 storage:
   banyandb:
-    host: ${SW_STORAGE_BANYANDB_HOST:127.0.0.1}
-    port: ${SW_STORAGE_BANYANDB_PORT:17912}
+    targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912}
     maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:5000}
     flushInterval: ${SW_STORAGE_BANYANDB_FLUSH_INTERVAL:15}
     metricsShardsNumber: ${SW_STORAGE_BANYANDB_METRICS_SHARDS_NUMBER:1}
@@ -22,4 +23,4 @@ storage:
 ```
 
 For more details, please refer to the documents of [BanyanDB](https://skywalking.apache.org/docs/skywalking-banyandb/next/readme/)
-and [BanyanDB Java Client](https://github.com/apache/skywalking-banyandb-java-client) subprojects.
\ No newline at end of file
+and [BanyanDB Java Client](https://github.com/apache/skywalking-banyandb-java-client) subprojects.
diff --git a/oap-server-bom/pom.xml b/oap-server-bom/pom.xml
index 963bcedf28..32b421bcba 100644
--- a/oap-server-bom/pom.xml
+++ b/oap-server-bom/pom.xml
@@ -72,7 +72,7 @@
         <awaitility.version>3.0.0</awaitility.version>
         <httpcore.version>4.4.13</httpcore.version>
         <commons-compress.version>1.21</commons-compress.version>
-        <banyandb-java-client.version>0.4.0</banyandb-java-client.version>
+        <banyandb-java-client.version>0.5.0</banyandb-java-client.version>
         <kafka-clients.version>2.8.1</kafka-clients.version>
         <spring-kafka-test.version>2.4.6.RELEASE</spring-kafka-test.version>
         <consul.client.version>1.5.3</consul.client.version>
diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml
index 7ac32a390e..a38a532311 100644
--- a/oap-server/server-starter/src/main/resources/application.yml
+++ b/oap-server/server-starter/src/main/resources/application.yml
@@ -213,8 +213,7 @@ storage:
     maxSizeOfBatchSql: ${SW_STORAGE_MAX_SIZE_OF_BATCH_SQL:2000}
     asyncBatchPersistentPoolSize: ${SW_STORAGE_ASYNC_BATCH_PERSISTENT_POOL_SIZE:4}
   banyandb:
-    host: ${SW_STORAGE_BANYANDB_HOST:127.0.0.1}
-    port: ${SW_STORAGE_BANYANDB_PORT:17912}
+    targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912}
     maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:5000}
     flushInterval: ${SW_STORAGE_BANYANDB_FLUSH_INTERVAL:15}
     metricsShardsNumber: ${SW_STORAGE_BANYANDB_METRICS_SHARDS_NUMBER:1}
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageClient.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageClient.java
index 136786865d..f09fb7c312 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageClient.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageClient.java
@@ -55,8 +55,8 @@ public class BanyanDBStorageClient implements Client, HealthCheckable {
     final BanyanDBClient client;
     private final DelegatedHealthChecker healthChecker = new DelegatedHealthChecker();
 
-    public BanyanDBStorageClient(String host, int port) {
-        this.client = new BanyanDBClient(host, port);
+    public BanyanDBStorageClient(String... targets) {
+        this.client = new BanyanDBClient(targets);
     }
 
     @Override
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java
index 39267d1b7f..91617f64da 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java
@@ -18,6 +18,8 @@
 
 package org.apache.skywalking.oap.server.storage.plugin.banyandb;
 
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
 import lombok.Getter;
 import lombok.Setter;
 import org.apache.skywalking.oap.server.library.module.ModuleConfig;
@@ -25,8 +27,12 @@ import org.apache.skywalking.oap.server.library.module.ModuleConfig;
 @Getter
 @Setter
 public class BanyanDBStorageConfig extends ModuleConfig {
-    private String host = "127.0.0.1";
-    private int port = 17912;
+    /**
+     * A comma-separated list of BanyanDB targets.
+     *
+     * @since 9.7.0
+     */
+    private String targets = "127.0.0.1:17912";
     /**
      * The maximum size of write entities in a single batch write call.
      */
@@ -96,4 +102,8 @@ public class BanyanDBStorageConfig extends ModuleConfig {
      * @since 9.4.0
      */
     private String specificGroupSettings;
+
+    public String[] getTargetArray() {
+        return Iterables.toArray(Splitter.on(",").omitEmptyStrings().trimResults().split(this.targets), String.class);
+    }
 }
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java
index d0ffcd6137..eed76a31b4 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java
@@ -118,7 +118,7 @@ public class BanyanDBStorageProvider extends ModuleProvider {
     public void prepare() throws ServiceNotProvidedException, ModuleStartException {
         this.registerServiceImplementation(StorageBuilderFactory.class, new StorageBuilderFactory.Default());
 
-        this.client = new BanyanDBStorageClient(config.getHost(), config.getPort());
+        this.client = new BanyanDBStorageClient(config.getTargetArray());
         this.modelInstaller = new BanyanDBIndexInstaller(client, getManager(), this.config);
 
         // Stream
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBMetadataQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBMetadataQueryDAO.java
index 21aede89a8..ff8df6cd30 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBMetadataQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBMetadataQueryDAO.java
@@ -19,10 +19,12 @@
 package org.apache.skywalking.oap.server.storage.plugin.banyandb.measure;
 
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
 import com.google.gson.Gson;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.skywalking.banyandb.v1.client.AbstractCriteria;
 import org.apache.skywalking.banyandb.v1.client.DataPoint;
 import org.apache.skywalking.banyandb.v1.client.MeasureQuery;
 import org.apache.skywalking.banyandb.v1.client.MeasureQueryResponse;
@@ -92,12 +94,12 @@ public class BanyanDBMetadataQueryDAO extends AbstractBanyanDBDAO implements IMe
     @Override
     public List<Service> listServices() throws IOException {
         MeasureQueryResponse resp = query(ServiceTraffic.INDEX_NAME,
-            SERVICE_TRAFFIC_TAGS,
-            Collections.emptySet(), new QueryBuilder<MeasureQuery>() {
-                @Override
-                protected void apply(MeasureQuery query) {
-                }
-            });
+                SERVICE_TRAFFIC_TAGS,
+                Collections.emptySet(), new QueryBuilder<MeasureQuery>() {
+                    @Override
+                    protected void apply(MeasureQuery query) {
+                    }
+                });
 
         final List<Service> services = new ArrayList<>();
         MetadataRegistry.Schema schema = MetadataRegistry.INSTANCE.findMetadata(ServiceTraffic.INDEX_NAME, DownSampling.Minute);
@@ -144,8 +146,8 @@ public class BanyanDBMetadataQueryDAO extends AbstractBanyanDBDAO implements IMe
                 new QueryBuilder<MeasureQuery>() {
                     @Override
                     protected void apply(MeasureQuery query) {
-                            query.and(eq(InstanceTraffic.SERVICE_ID, id.getServiceId()))
-                                    .and(eq(InstanceTraffic.NAME, id.getName()));
+                        query.and(eq(InstanceTraffic.SERVICE_ID, id.getServiceId()))
+                                .and(eq(InstanceTraffic.NAME, id.getName()));
                     }
                 });
         MetadataRegistry.Schema schema = MetadataRegistry.INSTANCE.findMetadata(InstanceTraffic.INDEX_NAME, DownSampling.Minute);
@@ -155,14 +157,19 @@ public class BanyanDBMetadataQueryDAO extends AbstractBanyanDBDAO implements IMe
     @Override
     public List<ServiceInstance> getInstances(List<String> instanceIds) throws IOException {
         MeasureQueryResponse resp = query(InstanceTraffic.INDEX_NAME,
-            INSTANCE_TRAFFIC_TAGS,
-            Collections.emptySet(),
-            new QueryBuilder<MeasureQuery>() {
-                @Override
-                protected void apply(MeasureQuery query) {
-                    query.and(in(InstanceTraffic.ID, instanceIds));
-                }
-            });
+                INSTANCE_TRAFFIC_TAGS,
+                Collections.emptySet(),
+                new QueryBuilder<MeasureQuery>() {
+                    @Override
+                    protected void apply(MeasureQuery query) {
+                        List<AbstractCriteria> instanceRelationsQueryConditions = new ArrayList<>(instanceIds.size());
+                        for (final String instanceId : instanceIds) {
+                            final IDManager.ServiceInstanceID.InstanceIDDefinition def = IDManager.ServiceInstanceID.analysisId(instanceId);
+                            and(Lists.newArrayList(eq(InstanceTraffic.SERVICE_ID, def.getServiceId()), eq(InstanceTraffic.NAME, def.getName())));
+                        }
+                        query.criteria(or(instanceRelationsQueryConditions));
+                    }
+                });
         MetadataRegistry.Schema schema = MetadataRegistry.INSTANCE.findMetadata(InstanceTraffic.INDEX_NAME, DownSampling.Minute);
         return resp.getDataPoints().stream().map(e -> buildInstance(e, schema)).collect(Collectors.toList());
     }
diff --git a/test/e2e-v2/script/docker-compose/base-compose.yml b/test/e2e-v2/script/docker-compose/base-compose.yml
index 1a78645d3e..4551c0c1bd 100644
--- a/test/e2e-v2/script/docker-compose/base-compose.yml
+++ b/test/e2e-v2/script/docker-compose/base-compose.yml
@@ -33,7 +33,7 @@ services:
       SW_STORAGE_ES_CLUSTER_NODES: es:9200
       SW_JDBC_URL: jdbc:mysql://mysql:3306/swtest?allowMultiQueries=true
       SW_STORAGE_INFLUXDB_URL: http://influxdb:8086
-      SW_STORAGE_BANYANDB_HOST: "banyandb"
+      SW_STORAGE_BANYANDB_TARGETS: banyandb:17912
       SW_CONFIG_ETCD_PERIOD: 1
       SW_CONFIG_ETCD_ENDPOINTS: http://etcd:2379
       SW_CLUSTER_ETCD_ENDPOINTS: http://etcd:2379
diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env
index 2c5956f986..a8431f5296 100644
--- a/test/e2e-v2/script/env
+++ b/test/e2e-v2/script/env
@@ -23,6 +23,6 @@ SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449
 SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016
 SW_KUBERNETES_COMMIT_SHA=e2c61c6774cf377b23516fca6f8a1e119d3191c5
 SW_ROVER_COMMIT=fc8d074c6d34ecfee585a7097cbd5aef1ca680a5
-SW_BANYANDB_COMMIT=df34a83663d66af9687ff484884ceae669c83980
+SW_BANYANDB_COMMIT=63be6e80bef82ea274d7352cc7a461b757d28355
 
 SW_CTL_COMMIT=9d2d0edccda2afe5cf24f1e632142f40e80e8fa4