You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2022/04/11 13:54:56 UTC
[skywalking] 01/01: Move ElasticSearch and BanyanDB specific configurations out of column.
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch banyandb-mata
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 145b2a15b732d9c3b9a83e65a76a75276e319011
Author: Wu Sheng <wu...@foxmail.com>
AuthorDate: Mon Apr 11 21:54:45 2022 +0800
Move ElasticSearch and BanyanDB specific configurations out of column.
---
.../org/apache/skywalking/oal/rt/OALRuntime.java | 4 +-
.../oap/server/core/alarm/AlarmRecord.java | 4 +-
.../analysis/manual/endpoint/EndpointTraffic.java | 4 +-
.../analysis/manual/log/AbstractLogRecord.java | 13 ++++-
.../manual/networkalias/NetworkAddressAlias.java | 4 +-
.../EndpointRelationServerSideMetrics.java | 4 +-
.../ServiceInstanceRelationServerSideMetrics.java | 4 +-
.../service/ServiceRelationServerSideMetrics.java | 4 +-
.../analysis/manual/segment/SegmentRecord.java | 12 +++-
.../analysis/manual/service/ServiceTraffic.java | 4 +-
.../analysis/meter/function/HistogramFunction.java | 4 +-
.../meter/function/PercentileFunction.java | 4 +-
.../analysis/meter/function/avg/AvgFunction.java | 4 +-
.../meter/function/avg/AvgHistogramFunction.java | 4 +-
.../avg/AvgHistogramPercentileFunction.java | 4 +-
.../meter/function/avg/AvgLabeledFunction.java | 4 +-
.../meter/function/latest/LatestFunction.java | 4 +-
.../analysis/meter/function/sum/SumFunction.java | 4 +-
.../manual/errorlog/BrowserErrorLogRecord.java | 4 +-
.../ebpf/storage/EBPFProfilingDataRecord.java | 4 +-
.../storage/annotation/BanyanDBGlobalIndex.java | 46 +++++++++++++++
.../storage/annotation/BanyanDBShardingKey.java | 59 +++++++++++++++++++
.../oap/server/core/storage/annotation/Column.java | 56 ------------------
.../annotation/ElasticSearchMatchQuery.java | 64 +++++++++++++++++++++
.../core/storage/model/BanyanDBExtension.java | 67 ++++++++++++++++++++++
.../core/storage/model/ElasticSearchExtension.java | 42 ++++++++++++++
.../oap/server/core/storage/model/ModelColumn.java | 25 ++++----
.../server/core/storage/model/StorageModels.java | 47 ++++++++++++---
.../server/core/storage/model/ModelColumnTest.java | 30 ++++++----
.../plugin/elasticsearch/base/AnalyzerSetting.java | 12 ++--
.../elasticsearch/base/StorageEsInstaller.java | 9 +--
.../storage/plugin/zipkin/ZipkinSpanRecord.java | 12 ++--
32 files changed, 440 insertions(+), 126 deletions(-)
diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java
index 4ccf35f460..98b67d02d6 100644
--- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java
+++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java
@@ -71,6 +71,7 @@ import org.apache.skywalking.oap.server.core.source.oal.rt.metrics.MetricClassPa
import org.apache.skywalking.oap.server.core.source.oal.rt.metrics.builder.MetricBuilderClassPackageHolder;
import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory;
import org.apache.skywalking.oap.server.core.storage.StorageException;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import org.apache.skywalking.oap.server.library.util.ResourceUtils;
@@ -269,7 +270,8 @@ public class OALRuntime implements OALEngine {
}
if (field.isID()) {
// Add shardingKeyIdx = 0 to column annotation.
- columnAnnotation.addMemberValue("shardingKeyIdx", new IntegerMemberValue(constPool, 0));
+ Annotation banyanShardingKeyAnnotation = new Annotation(BanyanDBShardingKey.class.getName(), constPool);
+ banyanShardingKeyAnnotation.addMemberValue("index", new IntegerMemberValue(constPool, 0));
}
annotationsAttribute.addAnnotation(columnAnnotation);
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java
index 8c292dc759..b55585b7ad 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java
@@ -29,6 +29,7 @@ import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcess
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.source.ScopeDeclaration;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter;
@@ -68,7 +69,8 @@ public class AlarmRecord extends Record {
private String id1;
@Column(columnName = START_TIME)
private long startTime;
- @Column(columnName = ALARM_MESSAGE, matchQuery = true)
+ @Column(columnName = ALARM_MESSAGE)
+ @ElasticSearchMatchQuery
private String alarmMessage;
@Column(columnName = RULE_NAME)
private String ruleName;
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java
index 432c7503b5..62da3fcfbb 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java
@@ -31,6 +31,7 @@ import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProces
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
@@ -52,7 +53,8 @@ public class EndpointTraffic extends Metrics {
private String serviceId;
@Setter
@Getter
- @Column(columnName = NAME, matchQuery = true)
+ @Column(columnName = NAME)
+ @ElasticSearchMatchQuery
private String name = Const.EMPTY_STRING;
@Override
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java
index fa405a0906..bc59aef887 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java
@@ -25,7 +25,10 @@ import org.apache.skywalking.oap.server.core.UnexpectedException;
import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.query.type.ContentType;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBGlobalIndex;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter;
@@ -47,11 +50,13 @@ public abstract class AbstractLogRecord extends Record {
@Setter
@Getter
- @Column(columnName = SERVICE_ID, shardingKeyIdx = 0)
+ @Column(columnName = SERVICE_ID)
+ @BanyanDBShardingKey(index = 0)
private String serviceId;
@Setter
@Getter
- @Column(columnName = SERVICE_INSTANCE_ID, shardingKeyIdx = 1)
+ @Column(columnName = SERVICE_INSTANCE_ID)
+ @BanyanDBShardingKey(index = 1)
private String serviceInstanceId;
@Setter
@Getter
@@ -64,6 +69,7 @@ public abstract class AbstractLogRecord extends Record {
@Setter
@Getter
@Column(columnName = TRACE_SEGMENT_ID, length = 150)
+ @BanyanDBGlobalIndex(extraFields = {SPAN_ID})
private String traceSegmentId;
@Setter
@Getter
@@ -75,7 +81,8 @@ public abstract class AbstractLogRecord extends Record {
private int contentType = ContentType.NONE.value();
@Setter
@Getter
- @Column(columnName = CONTENT, length = 1_000_000, matchQuery = true, analyzer = Column.AnalyzerType.OAP_LOG_ANALYZER)
+ @Column(columnName = CONTENT, length = 1_000_000)
+ @ElasticSearchMatchQuery(analyzer = ElasticSearchMatchQuery.AnalyzerType.OAP_LOG_ANALYZER)
private String content;
@Setter
@Getter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/networkalias/NetworkAddressAlias.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/networkalias/NetworkAddressAlias.java
index c07c3c33f8..bddf8bda14 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/networkalias/NetworkAddressAlias.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/networkalias/NetworkAddressAlias.java
@@ -28,6 +28,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.ScopeDeclaration;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -51,7 +52,8 @@ public class NetworkAddressAlias extends Metrics {
@Setter
@Getter
- @Column(columnName = ADDRESS, shardingKeyIdx = 0)
+ @Column(columnName = ADDRESS)
+ @BanyanDBShardingKey(index = 0)
private String address;
@Setter
@Getter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/endpoint/EndpointRelationServerSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/endpoint/EndpointRelationServerSideMetrics.java
index f07aada393..08863f11b0 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/endpoint/EndpointRelationServerSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/endpoint/EndpointRelationServerSideMetrics.java
@@ -27,6 +27,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -58,7 +59,8 @@ public class EndpointRelationServerSideMetrics extends Metrics {
private int componentId;
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Override
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java
index 484f5de214..dd674cef84 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java
@@ -27,6 +27,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -68,7 +69,8 @@ public class ServiceInstanceRelationServerSideMetrics extends Metrics {
private int componentId;
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Override
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java
index b68ddddb69..435ab6997a 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java
@@ -27,6 +27,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -58,7 +59,8 @@ public class ServiceRelationServerSideMetrics extends Metrics {
private int componentId;
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Override
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
index 90d13792c8..d25091779c 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
@@ -26,6 +26,8 @@ import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBGlobalIndex;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
@@ -56,14 +58,17 @@ public class SegmentRecord extends Record {
@Setter
@Getter
@Column(columnName = TRACE_ID, length = 150)
+ @BanyanDBGlobalIndex(extraFields = {})
private String traceId;
@Setter
@Getter
- @Column(columnName = SERVICE_ID, shardingKeyIdx = 0)
+ @Column(columnName = SERVICE_ID)
+ @BanyanDBShardingKey(index = 0)
private String serviceId;
@Setter
@Getter
- @Column(columnName = SERVICE_INSTANCE_ID, shardingKeyIdx = 1)
+ @Column(columnName = SERVICE_INSTANCE_ID)
+ @BanyanDBShardingKey(index = 1)
private String serviceInstanceId;
@Setter
@Getter
@@ -79,7 +84,8 @@ public class SegmentRecord extends Record {
private int latency;
@Setter
@Getter
- @Column(columnName = IS_ERROR, shardingKeyIdx = 2)
+ @Column(columnName = IS_ERROR)
+ @BanyanDBShardingKey(index = 2)
private int isError;
@Setter
@Getter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java
index 0d56840c18..6ad15a8f4b 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java
@@ -31,6 +31,7 @@ import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProces
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
@@ -60,7 +61,8 @@ public class ServiceTraffic extends Metrics {
@Setter
@Getter
- @Column(columnName = NAME, matchQuery = true)
+ @Column(columnName = NAME)
+ @ElasticSearchMatchQuery
private String name = Const.EMPTY_STRING;
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/HistogramFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/HistogramFunction.java
index e0ceaa962e..0974829ec7 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/HistogramFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/HistogramFunction.java
@@ -31,6 +31,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.query.type.Bucket;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -48,7 +49,8 @@ public abstract class HistogramFunction extends Meter implements AcceptableValue
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java
index 70ec29ca14..8d5760e4ce 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java
@@ -37,6 +37,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.MultiIntValuesHold
import org.apache.skywalking.oap.server.core.analysis.metrics.PercentileMetrics;
import org.apache.skywalking.oap.server.core.query.type.Bucket;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -55,7 +56,8 @@ public abstract class PercentileFunction extends Meter implements AcceptableValu
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java
index 6476746865..7b1bb9d97a 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java
@@ -36,6 +36,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Entranc
import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.SourceFrom;
import org.apache.skywalking.oap.server.core.query.sql.Function;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -50,7 +51,8 @@ public abstract class AvgFunction extends Meter implements AcceptableValue<Long>
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
/**
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java
index 0222c971e2..8a88a6e17e 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java
@@ -34,6 +34,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.query.type.Bucket;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -59,7 +60,8 @@ public abstract class AvgHistogramFunction extends Meter implements AcceptableVa
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java
index f785c39a81..794f8668d7 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java
@@ -43,6 +43,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.metrics.MultiIntValuesHolder;
import org.apache.skywalking.oap.server.core.query.type.Bucket;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -74,7 +75,8 @@ public abstract class AvgHistogramPercentileFunction extends Meter implements Ac
@Setter
@Getter
- @Column(columnName = ENTITY_ID, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java
index 8721a77cda..7a44ee569d 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java
@@ -34,6 +34,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.analysis.metrics.LabeledValueHolder;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -48,7 +49,8 @@ public abstract class AvgLabeledFunction extends Meter implements AcceptableValu
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
/**
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/latest/LatestFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/latest/LatestFunction.java
index 354988e646..c791422ab5 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/latest/LatestFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/latest/LatestFunction.java
@@ -35,6 +35,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Entranc
import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.SourceFrom;
import org.apache.skywalking.oap.server.core.query.sql.Function;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -47,7 +48,8 @@ public abstract class LatestFunction extends Meter implements AcceptableValue<Lo
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
/**
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumFunction.java
index 0649939476..c97b92f51c 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumFunction.java
@@ -35,6 +35,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Entranc
import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.SourceFrom;
import org.apache.skywalking.oap.server.core.query.sql.Function;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -47,7 +48,8 @@ public abstract class SumFunction extends Meter implements AcceptableValue<Long>
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java
index df4e49b435..df12484b0b 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java
@@ -23,6 +23,7 @@ import org.apache.skywalking.oap.server.core.analysis.Stream;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
@@ -54,7 +55,8 @@ public class BrowserErrorLogRecord extends Record {
@Setter
@Getter
- @Column(columnName = SERVICE_ID, shardingKeyIdx = 0)
+ @Column(columnName = SERVICE_ID)
+ @BanyanDBShardingKey(index = 0)
private String serviceId;
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingDataRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingDataRecord.java
index dc2433bd7f..98b1519cef 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingDataRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingDataRecord.java
@@ -24,6 +24,7 @@ import lombok.Data;
import org.apache.skywalking.oap.server.core.analysis.Stream;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -48,7 +49,8 @@ public class EBPFProfilingDataRecord extends Record {
public static final String STACKS_BINARY = "dump_binary";
public static final String UPLOAD_TIME = "upload_time";
- @Column(columnName = TASK_ID, length = 600, shardingKeyIdx = 0)
+ @Column(columnName = TASK_ID, length = 600)
+ @BanyanDBShardingKey(index = 0)
private String taskId;
@Column(columnName = SCHEDULE_ID, length = 600)
private String scheduleId;
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBGlobalIndex.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBGlobalIndex.java
new file mode 100644
index 0000000000..a8dc1b761f
--- /dev/null
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBGlobalIndex.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.storage.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * BanyanDBGlobalIndex declares advanced global index, which are only available in BanyanDB.
+ *
+ * Global index should only be considered if a column value has a huge value candidates, but we will need a direct equal
+ * query without timestamp.
+ * The typical global index is designed for huge candidate of indexed values,
+ * such as `trace ID` or `segment ID + span ID`
+ *
+ * Only work with {@link Column}
+ */
+@Target({ElementType.FIELD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface BanyanDBGlobalIndex {
+ /**
+ * The current column should be indexed through global index.
+ *
+ * @return empty array if only the current column should be indexed in global index. Or list of column names if this
+ * global index includes multiple columns.
+ */
+ String[] extraFields();
+}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBShardingKey.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBShardingKey.java
new file mode 100644
index 0000000000..8c8e21a172
--- /dev/null
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBShardingKey.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.storage.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Sharding key is used to group time series data per metric of one entity in one place (same sharding and/or same
+ * row for column-oriented database).
+ * For example,
+ * ServiceA's traffic gauge, service call per minute, includes following timestamp values, then it should be sharded
+ * by service ID
+ * [ServiceA(encoded ID): 01-28 18:30 values-1, 01-28 18:31 values-2, 01-28 18:32 values-3, 01-28 18:32 values-4]
+ *
+ * BanyanDB is the 1st storage implementation supporting this. It would make continuous time series metrics stored
+ * closely and compressed better.
+ *
+ * 1. One entity could have multiple sharding keys
+ * 2. If no column is appointed for this, {@link org.apache.skywalking.oap.server.core.storage.StorageData#id}
+ * would be used by the storage implementation accordingly.
+ *
+ * NOTICE, this sharding concept is NOT just for splitting data into different database instances or physical
+ * files.
+ *
+ * Only work with {@link Column}
+ *
+ * @return non-negative if this column be used for sharding. -1 means not as a sharding key
+ * @since 9.1.0 created as a new annotation.
+ * @since 9.0.0 added in {@link Column}
+ */
+@Target({ElementType.FIELD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface BanyanDBShardingKey {
+ /**
+ * Relative entity tag
+ *
+ * @return index, from zero.
+ */
+ int index() default -1;
+}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java
index 077fdfe6dd..44d2305024 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java
@@ -48,11 +48,6 @@ public @interface Column {
*/
int defaultValue() default 0;
- /**
- * Match query means using analyzer(if storage have) to do key word match query.
- */
- boolean matchQuery() default false;
-
/**
* The column is just saved, never used in query.
*/
@@ -94,57 +89,6 @@ public @interface Column {
*/
ValueDataType dataType() default ValueDataType.NOT_VALUE;
- /**
- * The storage analyzer mode.
- *
- * @since 8.4.0
- */
- AnalyzerType analyzer() default AnalyzerType.OAP_ANALYZER;
-
- /**
- * Sharding key is used to group time series data per metric of one entity in one place (same sharding and/or same
- * row for column-oriented database).
- * For example,
- * ServiceA's traffic gauge, service call per minute, includes following timestamp values, then it should be sharded
- * by service ID
- * [ServiceA(encoded ID): 01-28 18:30 values-1, 01-28 18:31 values-2, 01-28 18:32 values-3, 01-28 18:32 values-4]
- *
- * BanyanDB is the 1st storage implementation supporting this. It would make continuous time series metrics stored
- * closely and compressed better.
- *
- * 1. One entity could have multiple sharding keys
- * 2. If no column is appointed for this, {@link org.apache.skywalking.oap.server.core.storage.StorageData#id}
- * would be used by the storage implementation accordingly.
- *
- * NOTICE, this sharding concept is NOT just for splitting data into different database instances or physical
- * files.
- *
- * @return non-negative if this column be used for sharding. -1 means not as a sharding key
- * @since 9.0.0
- */
- int shardingKeyIdx() default -1;
-
- /**
- * The analyzer declares the text analysis mode.
- */
- enum AnalyzerType {
- /**
- * The default analyzer.
- */
- OAP_ANALYZER("oap_analyzer"),
- /**
- * The log analyzer.
- */
- OAP_LOG_ANALYZER("oap_log_analyzer");
-
- @Getter
- private final String name;
-
- AnalyzerType(final String name) {
- this.name = name;
- }
- }
-
/**
* ValueDataType represents the data structure of value column. The persistent way of the value column determine the
* available ways to query the data.
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/ElasticSearchMatchQuery.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/ElasticSearchMatchQuery.java
new file mode 100644
index 0000000000..19084754de
--- /dev/null
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/ElasticSearchMatchQuery.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.storage.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import lombok.Getter;
+
+/**
+ * Match query is designed for ElasticSearch match query with specific analyzer. It is a fuzzy query implementation
+ * powered by analyzer.
+ *
+ * @since 9.1.0 This used to be {@link Column}'s matchQuery and analyzer attributes.
+ */
+@Target({ElementType.FIELD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface ElasticSearchMatchQuery {
+ /**
+ * The storage analyzer mode.
+ *
+ * @since 9.1.0 created as a new annotation.
+ * @since 8.4.0 added in {@link Column}
+ */
+ AnalyzerType analyzer() default AnalyzerType.OAP_ANALYZER;
+
+ /**
+ * The analyzer declares the text analysis mode.
+ */
+ enum AnalyzerType {
+ /**
+ * The default analyzer.
+ */
+ OAP_ANALYZER("oap_analyzer"),
+ /**
+ * The log analyzer.
+ */
+ OAP_LOG_ANALYZER("oap_log_analyzer");
+
+ @Getter
+ private final String name;
+
+ AnalyzerType(final String name) {
+ this.name = name;
+ }
+ }
+}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java
new file mode 100644
index 0000000000..2a8c82ee26
--- /dev/null
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.storage.model;
+
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+/**
+ * BanyanDBExtension represents extra metadata for columns, but specific for BanyanDB usages.
+ *
+ * @since 9.1.0
+ */
+@Getter
+@RequiredArgsConstructor
+public class BanyanDBExtension {
+ /**
+ * Sharding key is used to group time series data per metric of one entity. See {@link
+ * org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey#index()}.
+ *
+ * @since 9.1.0 moved into BanyanDBExtension
+ * @since 9.0.0 added into {@link ModelColumn}
+ */
+ private final int shardingKeyIdx;
+
+ /**
+ * Global index column values has 3 conditions
+ * 1. NULL, this column should be as a global index.
+ * 2. Empty array(declared by @BanyanDBGlobalIndex(extraFields = {}) in codes) represents this single column should
+ * be a global index.
+ * 3. Not empty array(declared by @BanyanDBGlobalIndex(extraFields = {"col1", "col2"}) in codes) represents this
+ * column and other declared columns should be as a global index. The values of these columns should be joint by
+ * underline(_)
+ *
+ * @since 9.1.0
+ */
+ private final String[] globalIndexColumns;
+
+ /**
+ * @return true if this column is a part of sharding key
+ */
+ public boolean isShardingKey() {
+ return this.shardingKeyIdx > -1;
+ }
+
+ /**
+ * @return null or array, see {@link #globalIndexColumns} for more details.
+ */
+ public boolean isGlobalIndexing() {
+ return globalIndexColumns != null;
+ }
+}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ElasticSearchExtension.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ElasticSearchExtension.java
new file mode 100644
index 0000000000..4ec91d990b
--- /dev/null
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ElasticSearchExtension.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.storage.model;
+
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
+
+/**
+ * ElasticSearchExtension represents extra metadata for columns, but specific for ElasticSearch usages.
+ *
+ * @since 9.1.0
+ */
+@Getter
+@RequiredArgsConstructor
+public class ElasticSearchExtension {
+ /**
+ * The analyzer policy appointed to fuzzy query, especially for ElasticSearch.
+ * When it is null, it means no need to build match query, no `copy_to` column, and no analyzer assigned.
+ */
+ private final ElasticSearchMatchQuery.AnalyzerType analyzer;
+
+ public boolean needMatchQuery() {
+ return analyzer != null;
+ }
+}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java
index 9d381f19ae..8959d0bbee 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java
@@ -22,7 +22,6 @@ import java.lang.reflect.Type;
import lombok.Getter;
import lombok.ToString;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
-import org.apache.skywalking.oap.server.core.storage.annotation.Column;
@Getter
@ToString
@@ -30,7 +29,6 @@ public class ModelColumn {
private final ColumnName columnName;
private final Class<?> type;
private final Type genericType;
- private final boolean matchQuery;
/**
* Storage this column for query result, but can't be as a condition . Conflict with {@link #indexOnly}
*/
@@ -45,33 +43,34 @@ public class ModelColumn {
* The max length of column value for length sensitive database.
*/
private final int length;
+
/**
- * The analyzer policy appointed to fuzzy query, especially for ElasticSearch
+ * Hold configurations especially for ElasticSearch
+ *
+ * @since 9.1.0
*/
- private final Column.AnalyzerType analyzer;
+ private ElasticSearchExtension elasticSearchExtension;
/**
- * Sharding key is used to group time series data per metric of one entity. See {@link Column#shardingKeyIdx()}.
+ * Hold configurations especially for BanyanDB relevant
*
- * @since 9.0.0
+ * @since 9.1.0
*/
- private int shardingKeyIdx;
+ private BanyanDBExtension banyanDBExtension;
public ModelColumn(ColumnName columnName,
Class<?> type,
Type genericType,
- boolean matchQuery,
boolean storageOnly,
boolean indexOnly,
boolean isValue,
int length,
- Column.AnalyzerType analyzer,
- int shardingKeyIdx) {
+ ElasticSearchExtension elasticSearchExtension,
+ BanyanDBExtension banyanDBExtension) {
this.columnName = columnName;
this.type = type;
this.genericType = genericType;
- this.matchQuery = matchQuery;
this.length = length;
- this.analyzer = analyzer;
+ this.elasticSearchExtension = elasticSearchExtension;
/*
* byte[] and {@link IntKeyLongValueHashMap} could never be query.
*/
@@ -90,6 +89,6 @@ public class ModelColumn {
"The column " + columnName + " can't be defined as both indexOnly and storageOnly.");
}
this.indexOnly = indexOnly;
- this.shardingKeyIdx = shardingKeyIdx;
+ this.banyanDBExtension = banyanDBExtension;
}
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
index 8aebfb1dc4..fe0e44d084 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
@@ -27,7 +27,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.core.analysis.FunctionCategory;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.StorageException;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBGlobalIndex;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.core.storage.annotation.MultipleQueryUnifiedIndex;
import org.apache.skywalking.oap.server.core.storage.annotation.QueryUnifiedIndex;
import org.apache.skywalking.oap.server.core.storage.annotation.Storage;
@@ -62,8 +65,12 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
checker.check(storage.getModelName());
Model model = new Model(
- storage.getModelName(), modelColumns, extraQueryIndices, scopeId,
- storage.getDownsampling(), record,
+ storage.getModelName(),
+ modelColumns,
+ extraQueryIndices,
+ scopeId,
+ storage.getDownsampling(),
+ record,
isSuperDatasetModel(aClass),
FunctionCategory.uniqueFunctionName(aClass),
storage.isTimeRelativeID()
@@ -129,13 +136,36 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
}
}
+ // ElasticSearch extension
+ final ElasticSearchMatchQuery elasticSearchAnalyzer = field.getAnnotation(
+ ElasticSearchMatchQuery.class);
+ ElasticSearchExtension elasticSearchExtension = new ElasticSearchExtension(
+ elasticSearchAnalyzer == null ? null : elasticSearchAnalyzer.analyzer()
+ );
+
+ // BanyanDB extension
+ final BanyanDBShardingKey banyanDBShardingKey = field.getAnnotation(BanyanDBShardingKey.class);
+ final BanyanDBGlobalIndex banyanDBGlobalIndex = field.getAnnotation(BanyanDBGlobalIndex.class);
+ BanyanDBExtension banyanDBExtension = new BanyanDBExtension(
+ banyanDBShardingKey == null ? -1 : banyanDBShardingKey.index(),
+ banyanDBGlobalIndex == null ? null : banyanDBGlobalIndex.extraFields()
+ );
+
final ModelColumn modelColumn = new ModelColumn(
- new ColumnName(modelName, column.columnName()), field.getType(), field.getGenericType(),
- column.matchQuery(), column.storageOnly(), column.indexOnly(), column.dataType().isValue(),
+ new ColumnName(
+ modelName,
+ column.columnName()
+ ),
+ field.getType(),
+ field.getGenericType(),
+ column.storageOnly(),
+ column.indexOnly(),
+ column.dataType().isValue(),
columnLength,
- column.analyzer(), column.shardingKeyIdx()
+ elasticSearchExtension,
+ banyanDBExtension
);
- if (column.shardingKeyIdx() > -1) {
+ if (banyanDBExtension.isShardingKey()) {
checker.accept(modelName, modelColumn);
}
@@ -197,7 +227,7 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
* @throws IllegalStateException if sharding key indices are conflicting.
*/
private void accept(String modelName, ModelColumn modelColumn) throws IllegalStateException {
- final int idx = modelColumn.getShardingKeyIdx();
+ final int idx = modelColumn.getBanyanDBExtension().getShardingKeyIdx();
while (idx + 1 > keys.size()) {
keys.add(null);
}
@@ -206,7 +236,8 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
throw new IllegalStateException(
modelName + "'s "
+ "Column [" + exist.getColumnName() + "] and column [" + modelColumn.getColumnName()
- + " are conflicting with sharding key index=" + modelColumn.getShardingKeyIdx());
+ + " are conflicting with sharding key index=" + modelColumn.getBanyanDBExtension()
+ .getShardingKeyIdx());
}
keys.set(idx, modelColumn);
}
diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java
index 90591834b3..c6f7ac430f 100644
--- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java
+++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java
@@ -19,31 +19,35 @@
package org.apache.skywalking.oap.server.core.storage.model;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
-import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.junit.Assert;
import org.junit.Test;
public class ModelColumnTest {
@Test
public void testColumnDefine() {
- ModelColumn column = new ModelColumn(new ColumnName("", "abc"), byte[].class, byte[].class, true,
+ ModelColumn column = new ModelColumn(new ColumnName("", "abc"), byte[].class, byte[].class,
false, false, true, 0,
- Column.AnalyzerType.OAP_ANALYZER, 0
+ new ElasticSearchExtension(
+ ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER),
+ new BanyanDBExtension(-1, null)
);
Assert.assertEquals(true, column.isStorageOnly());
Assert.assertEquals("abc", column.getColumnName().getName());
- column = new ModelColumn(new ColumnName("", "abc"), DataTable.class, DataTable.class, true,
+ column = new ModelColumn(new ColumnName("", "abc"), DataTable.class, DataTable.class,
false, false, true, 200,
- Column.AnalyzerType.OAP_ANALYZER, 0
+ new ElasticSearchExtension(ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER),
+ new BanyanDBExtension(-1, null)
);
Assert.assertEquals(true, column.isStorageOnly());
Assert.assertEquals("abc", column.getColumnName().getName());
Assert.assertEquals(200, column.getLength());
- column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class, true,
+ column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class,
false, false, true, 200,
- Column.AnalyzerType.OAP_ANALYZER, 0
+ new ElasticSearchExtension(ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER),
+ new BanyanDBExtension(-1, null)
);
Assert.assertEquals(false, column.isStorageOnly());
Assert.assertEquals("abc", column.getColumnName().getName());
@@ -52,16 +56,20 @@ public class ModelColumnTest {
@Test(expected = IllegalArgumentException.class)
public void testConflictDefinition() {
ModelColumn column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class,
- true, true, false, true, 200,
- Column.AnalyzerType.OAP_ANALYZER, 0
+ true, false, true, 200,
+ new ElasticSearchExtension(
+ ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER),
+ new BanyanDBExtension(-1, null)
);
}
@Test(expected = IllegalArgumentException.class)
public void testConflictDefinitionIndexOnly() {
ModelColumn column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class,
- true, true, true, false, 200,
- Column.AnalyzerType.OAP_ANALYZER, 0
+ true, true, false, 200,
+ new ElasticSearchExtension(
+ ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER),
+ new BanyanDBExtension(-1, null)
);
}
}
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/AnalyzerSetting.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/AnalyzerSetting.java
index 1de991027b..57e79ad0a4 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/AnalyzerSetting.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/AnalyzerSetting.java
@@ -27,7 +27,7 @@ import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.core.storage.StorageException;
-import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.StorageModuleElasticsearchConfig;
@Getter
@@ -79,18 +79,18 @@ public class AnalyzerSetting {
public enum Generator {
OAP_ANALYZER_SETTING_GENERATOR(
- Column.AnalyzerType.OAP_ANALYZER,
+ ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER,
config -> new Gson().fromJson(config.getOapAnalyzer(), AnalyzerSetting.class)
),
OAP_LOG_ANALYZER_SETTING_GENERATOR(
- Column.AnalyzerType.OAP_LOG_ANALYZER,
+ ElasticSearchMatchQuery.AnalyzerType.OAP_LOG_ANALYZER,
config -> new Gson().fromJson(config.getOapLogAnalyzer(), AnalyzerSetting.class)
);
- private final Column.AnalyzerType type;
+ private final ElasticSearchMatchQuery.AnalyzerType type;
private final GenerateAnalyzerSettingFunc func;
- Generator(final Column.AnalyzerType type,
+ Generator(final ElasticSearchMatchQuery.AnalyzerType type,
final GenerateAnalyzerSettingFunc func) {
this.type = type;
this.func = func;
@@ -100,7 +100,7 @@ public class AnalyzerSetting {
return this.func;
}
- public static Generator getGenerator(Column.AnalyzerType type) throws StorageException {
+ public static Generator getGenerator(ElasticSearchMatchQuery.AnalyzerType type) throws StorageException {
for (final Generator value : Generator.values()) {
if (value.type == type) {
return value;
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
index e77c666adb..afb540c827 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
-import org.apache.skywalking.oap.server.library.util.StringUtil;
import org.apache.skywalking.library.elasticsearch.response.Index;
import org.apache.skywalking.library.elasticsearch.response.IndexTemplate;
import org.apache.skywalking.library.elasticsearch.response.Mappings;
@@ -36,6 +35,7 @@ import org.apache.skywalking.oap.server.core.storage.model.ModelInstaller;
import org.apache.skywalking.oap.server.library.client.Client;
import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
+import org.apache.skywalking.oap.server.library.util.StringUtil;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.StorageModuleElasticsearchConfig;
@Slf4j
@@ -214,7 +214,8 @@ public class StorageEsInstaller extends ModelInstaller {
private Map getAnalyzerSetting(List<ModelColumn> analyzerTypes) throws StorageException {
AnalyzerSetting analyzerSetting = new AnalyzerSetting();
for (final ModelColumn column : analyzerTypes) {
- AnalyzerSetting setting = AnalyzerSetting.Generator.getGenerator(column.getAnalyzer())
+ AnalyzerSetting setting = AnalyzerSetting.Generator.getGenerator(
+ column.getElasticSearchExtension().getAnalyzer())
.getGenerateFunc()
.generate(config);
analyzerSetting.combine(setting);
@@ -227,7 +228,7 @@ public class StorageEsInstaller extends ModelInstaller {
Mappings.Source source = new Mappings.Source();
for (ModelColumn columnDefine : model.getColumns()) {
final String type = columnTypeEsMapping.transform(columnDefine.getType(), columnDefine.getGenericType());
- if (columnDefine.isMatchQuery()) {
+ if (columnDefine.getElasticSearchExtension().needMatchQuery()) {
String matchCName = MatchCNameBuilder.INSTANCE.build(columnDefine.getColumnName().getName());
Map<String, Object> originalColumn = new HashMap<>();
@@ -237,7 +238,7 @@ public class StorageEsInstaller extends ModelInstaller {
Map<String, Object> matchColumn = new HashMap<>();
matchColumn.put("type", "text");
- matchColumn.put("analyzer", columnDefine.getAnalyzer().getName());
+ matchColumn.put("analyzer", columnDefine.getElasticSearchExtension().getAnalyzer().getName());
properties.put(matchCName, matchColumn);
} else {
Map<String, Object> column = new HashMap<>();
diff --git a/oap-server/server-storage-plugin/storage-zipkin-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java b/oap-server/server-storage-plugin/storage-zipkin-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java
index ea4bbb13e4..16376804f8 100644
--- a/oap-server/server-storage-plugin/storage-zipkin-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java
+++ b/oap-server/server-storage-plugin/storage-zipkin-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java
@@ -25,6 +25,7 @@ import org.apache.skywalking.oap.server.core.analysis.Stream;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
@@ -60,15 +61,17 @@ public class ZipkinSpanRecord extends Record {
private String spanId;
@Setter
@Getter
- @Column(columnName = SERVICE_ID, shardingKeyIdx = 0)
+ @Column(columnName = SERVICE_ID)
+ @BanyanDBShardingKey(index = 0)
private String serviceId;
@Setter
@Getter
- @Column(columnName = SERVICE_INSTANCE_ID, shardingKeyIdx = 1)
+ @Column(columnName = SERVICE_INSTANCE_ID)
+ @BanyanDBShardingKey(index = 1)
private String serviceInstanceId;
@Setter
@Getter
- @Column(columnName = ENDPOINT_NAME, matchQuery = true)
+ @Column(columnName = ENDPOINT_NAME)
private String endpointName;
@Setter
@Getter
@@ -88,7 +91,8 @@ public class ZipkinSpanRecord extends Record {
private int latency;
@Setter
@Getter
- @Column(columnName = IS_ERROR, shardingKeyIdx = 2)
+ @Column(columnName = IS_ERROR)
+ @BanyanDBShardingKey(index = 2)
private int isError;
@Setter
@Getter