You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2022/04/12 00:21:58 UTC
[skywalking] branch master updated: Support BanyanDB global index for entities (#8852)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 6d9decddce Support BanyanDB global index for entities (#8852)
6d9decddce is described below
commit 6d9decddceaffd2f10a26cc3d818c6bac4e94ba2
Author: 吴晟 Wu Sheng <wu...@foxmail.com>
AuthorDate: Tue Apr 12 08:21:44 2022 +0800
Support BanyanDB global index for entities (#8852)
* Move ElasticSearch and BanyanDB specific configurations out of column.
* Support BanyanDB global index for entities. Log and Segment record entities declare this new feature.
* Remove unnecessary analyzer settings in columns of templates. Many were added due to analyzer's default value.
* Move SQLDatabase(H2/MySQL/PostgreSQL) specific configurations out of column.
---
docs/en/changes/changes.md | 10 ++-
.../org/apache/skywalking/oal/rt/OALRuntime.java | 4 +-
.../oap/server/core/alarm/AlarmRecord.java | 4 +-
.../analysis/manual/endpoint/EndpointTraffic.java | 4 +-
.../analysis/manual/log/AbstractLogRecord.java | 14 +++-
.../manual/networkalias/NetworkAddressAlias.java | 4 +-
.../EndpointRelationServerSideMetrics.java | 4 +-
.../ServiceInstanceRelationServerSideMetrics.java | 4 +-
.../service/ServiceRelationServerSideMetrics.java | 4 +-
.../analysis/manual/segment/SegmentRecord.java | 12 ++-
.../analysis/manual/service/ServiceTraffic.java | 4 +-
.../analysis/meter/function/HistogramFunction.java | 4 +-
.../meter/function/PercentileFunction.java | 4 +-
.../analysis/meter/function/avg/AvgFunction.java | 4 +-
.../meter/function/avg/AvgHistogramFunction.java | 4 +-
.../avg/AvgHistogramPercentileFunction.java | 4 +-
.../meter/function/avg/AvgLabeledFunction.java | 4 +-
.../meter/function/latest/LatestFunction.java | 4 +-
.../analysis/meter/function/sum/SumFunction.java | 4 +-
.../manual/errorlog/BrowserErrorLogRecord.java | 4 +-
.../ebpf/storage/EBPFProfilingDataRecord.java | 4 +-
.../storage/annotation/BanyanDBGlobalIndex.java | 46 +++++++++++
.../storage/annotation/BanyanDBShardingKey.java | 59 ++++++++++++++
.../oap/server/core/storage/annotation/Column.java | 56 -------------
.../annotation/ElasticSearchMatchQuery.java | 64 +++++++++++++++
.../core/storage/model/BanyanDBExtension.java | 67 ++++++++++++++++
.../storage/model/ElasticSearchExtension.java} | 32 ++++----
.../server/core/storage/model/ExtraQueryIndex.java | 56 -------------
.../oap/server/core/storage/model/Model.java | 3 -
.../oap/server/core/storage/model/ModelColumn.java | 33 +++++---
.../core/storage/model/SQLDatabaseExtension.java | 70 ++++++++++++++++
.../server/core/storage/model/StorageModels.java | 93 +++++++++++++++-------
.../server/core/storage/model/ModelColumnTest.java | 35 +++++---
...ndexTest.java => SQLDatabaseExtensionTest.java} | 8 +-
.../core/storage/model/StorageModelsTest.java | 4 +-
.../plugin/elasticsearch/base/AnalyzerSetting.java | 12 +--
.../elasticsearch/base/StorageEsInstaller.java | 12 ++-
.../elasticsearch/base/TimeSeriesUtilsTest.java | 6 +-
.../plugin/jdbc/mysql/MySQLTableInstaller.java | 33 ++++----
.../storage/plugin/zipkin/ZipkinSpanRecord.java | 12 ++-
40 files changed, 564 insertions(+), 245 deletions(-)
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 05812ca036..1e3679b17d 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -1,18 +1,24 @@
## 9.1.0
#### Project
+
* Upgrade zipkin to 2.23.16.
#### OAP Server
+
* Add component definition(ID=127) for `Apache ShenYu (incubating)`.
-* Fix Zipkin receiver: Decode spans error, missing `Layer` for V9 and wrong time bucket for generate Service and Endpoint.
+* Fix Zipkin receiver: Decode spans error, missing `Layer` for V9 and wrong time bucket for generate Service and
+ Endpoint.
+* [Refactor] Move SQLDatabase(H2/MySQL/PostgreSQL), ElasticSearch and BanyanDB specific configurations out of column.
+* Support BanyanDB global index for entities. Log and Segment record entities declare this new feature.
+* Remove unnecessary analyzer settings in columns of templates. Many were added due to analyzer's default value.
#### UI
+
* General service instance: move `Thread Pool` from JVM to Overview, fix `JVM GC Count` calculation.
* Add Apache ShenYu (incubating) component LOGO.
#### Documentation
-
All issues and pull requests are [here](https://github.com/apache/skywalking/milestone/128?closed=1)
diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java
index 4ccf35f460..98b67d02d6 100644
--- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java
+++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java
@@ -71,6 +71,7 @@ import org.apache.skywalking.oap.server.core.source.oal.rt.metrics.MetricClassPa
import org.apache.skywalking.oap.server.core.source.oal.rt.metrics.builder.MetricBuilderClassPackageHolder;
import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory;
import org.apache.skywalking.oap.server.core.storage.StorageException;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import org.apache.skywalking.oap.server.library.util.ResourceUtils;
@@ -269,7 +270,8 @@ public class OALRuntime implements OALEngine {
}
if (field.isID()) {
// Add shardingKeyIdx = 0 to column annotation.
- columnAnnotation.addMemberValue("shardingKeyIdx", new IntegerMemberValue(constPool, 0));
+ Annotation banyanShardingKeyAnnotation = new Annotation(BanyanDBShardingKey.class.getName(), constPool);
+ banyanShardingKeyAnnotation.addMemberValue("index", new IntegerMemberValue(constPool, 0));
}
annotationsAttribute.addAnnotation(columnAnnotation);
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java
index 8c292dc759..b55585b7ad 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmRecord.java
@@ -29,6 +29,7 @@ import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcess
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.source.ScopeDeclaration;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter;
@@ -68,7 +69,8 @@ public class AlarmRecord extends Record {
private String id1;
@Column(columnName = START_TIME)
private long startTime;
- @Column(columnName = ALARM_MESSAGE, matchQuery = true)
+ @Column(columnName = ALARM_MESSAGE)
+ @ElasticSearchMatchQuery
private String alarmMessage;
@Column(columnName = RULE_NAME)
private String ruleName;
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java
index 432c7503b5..62da3fcfbb 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpoint/EndpointTraffic.java
@@ -31,6 +31,7 @@ import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProces
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
@@ -52,7 +53,8 @@ public class EndpointTraffic extends Metrics {
private String serviceId;
@Setter
@Getter
- @Column(columnName = NAME, matchQuery = true)
+ @Column(columnName = NAME)
+ @ElasticSearchMatchQuery
private String name = Const.EMPTY_STRING;
@Override
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java
index fa405a0906..0ea0ee01ee 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/log/AbstractLogRecord.java
@@ -25,7 +25,10 @@ import org.apache.skywalking.oap.server.core.UnexpectedException;
import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.query.type.ContentType;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBGlobalIndex;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter;
@@ -47,11 +50,13 @@ public abstract class AbstractLogRecord extends Record {
@Setter
@Getter
- @Column(columnName = SERVICE_ID, shardingKeyIdx = 0)
+ @Column(columnName = SERVICE_ID)
+ @BanyanDBShardingKey(index = 0)
private String serviceId;
@Setter
@Getter
- @Column(columnName = SERVICE_INSTANCE_ID, shardingKeyIdx = 1)
+ @Column(columnName = SERVICE_INSTANCE_ID)
+ @BanyanDBShardingKey(index = 1)
private String serviceInstanceId;
@Setter
@Getter
@@ -60,10 +65,12 @@ public abstract class AbstractLogRecord extends Record {
@Setter
@Getter
@Column(columnName = TRACE_ID, length = 150)
+ @BanyanDBGlobalIndex(extraFields = {})
private String traceId;
@Setter
@Getter
@Column(columnName = TRACE_SEGMENT_ID, length = 150)
+ @BanyanDBGlobalIndex(extraFields = {SPAN_ID})
private String traceSegmentId;
@Setter
@Getter
@@ -75,7 +82,8 @@ public abstract class AbstractLogRecord extends Record {
private int contentType = ContentType.NONE.value();
@Setter
@Getter
- @Column(columnName = CONTENT, length = 1_000_000, matchQuery = true, analyzer = Column.AnalyzerType.OAP_LOG_ANALYZER)
+ @Column(columnName = CONTENT, length = 1_000_000)
+ @ElasticSearchMatchQuery(analyzer = ElasticSearchMatchQuery.AnalyzerType.OAP_LOG_ANALYZER)
private String content;
@Setter
@Getter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/networkalias/NetworkAddressAlias.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/networkalias/NetworkAddressAlias.java
index c07c3c33f8..bddf8bda14 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/networkalias/NetworkAddressAlias.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/networkalias/NetworkAddressAlias.java
@@ -28,6 +28,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.ScopeDeclaration;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -51,7 +52,8 @@ public class NetworkAddressAlias extends Metrics {
@Setter
@Getter
- @Column(columnName = ADDRESS, shardingKeyIdx = 0)
+ @Column(columnName = ADDRESS)
+ @BanyanDBShardingKey(index = 0)
private String address;
@Setter
@Getter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/endpoint/EndpointRelationServerSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/endpoint/EndpointRelationServerSideMetrics.java
index f07aada393..08863f11b0 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/endpoint/EndpointRelationServerSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/endpoint/EndpointRelationServerSideMetrics.java
@@ -27,6 +27,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -58,7 +59,8 @@ public class EndpointRelationServerSideMetrics extends Metrics {
private int componentId;
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Override
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java
index 484f5de214..dd674cef84 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java
@@ -27,6 +27,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -68,7 +69,8 @@ public class ServiceInstanceRelationServerSideMetrics extends Metrics {
private int componentId;
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Override
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java
index b68ddddb69..435ab6997a 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java
@@ -27,6 +27,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -58,7 +59,8 @@ public class ServiceRelationServerSideMetrics extends Metrics {
private int componentId;
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Override
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
index 90d13792c8..d25091779c 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
@@ -26,6 +26,8 @@ import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBGlobalIndex;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
@@ -56,14 +58,17 @@ public class SegmentRecord extends Record {
@Setter
@Getter
@Column(columnName = TRACE_ID, length = 150)
+ @BanyanDBGlobalIndex(extraFields = {})
private String traceId;
@Setter
@Getter
- @Column(columnName = SERVICE_ID, shardingKeyIdx = 0)
+ @Column(columnName = SERVICE_ID)
+ @BanyanDBShardingKey(index = 0)
private String serviceId;
@Setter
@Getter
- @Column(columnName = SERVICE_INSTANCE_ID, shardingKeyIdx = 1)
+ @Column(columnName = SERVICE_INSTANCE_ID)
+ @BanyanDBShardingKey(index = 1)
private String serviceInstanceId;
@Setter
@Getter
@@ -79,7 +84,8 @@ public class SegmentRecord extends Record {
private int latency;
@Setter
@Getter
- @Column(columnName = IS_ERROR, shardingKeyIdx = 2)
+ @Column(columnName = IS_ERROR)
+ @BanyanDBShardingKey(index = 2)
private int isError;
@Setter
@Getter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java
index 0d56840c18..6ad15a8f4b 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java
@@ -31,6 +31,7 @@ import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProces
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
@@ -60,7 +61,8 @@ public class ServiceTraffic extends Metrics {
@Setter
@Getter
- @Column(columnName = NAME, matchQuery = true)
+ @Column(columnName = NAME)
+ @ElasticSearchMatchQuery
private String name = Const.EMPTY_STRING;
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/HistogramFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/HistogramFunction.java
index e0ceaa962e..0974829ec7 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/HistogramFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/HistogramFunction.java
@@ -31,6 +31,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.query.type.Bucket;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -48,7 +49,8 @@ public abstract class HistogramFunction extends Meter implements AcceptableValue
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java
index 70ec29ca14..8d5760e4ce 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/PercentileFunction.java
@@ -37,6 +37,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.MultiIntValuesHold
import org.apache.skywalking.oap.server.core.analysis.metrics.PercentileMetrics;
import org.apache.skywalking.oap.server.core.query.type.Bucket;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -55,7 +56,8 @@ public abstract class PercentileFunction extends Meter implements AcceptableValu
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java
index 6476746865..7b1bb9d97a 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java
@@ -36,6 +36,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Entranc
import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.SourceFrom;
import org.apache.skywalking.oap.server.core.query.sql.Function;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -50,7 +51,8 @@ public abstract class AvgFunction extends Meter implements AcceptableValue<Long>
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
/**
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java
index 0222c971e2..8a88a6e17e 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramFunction.java
@@ -34,6 +34,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.query.type.Bucket;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -59,7 +60,8 @@ public abstract class AvgHistogramFunction extends Meter implements AcceptableVa
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java
index f785c39a81..794f8668d7 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgHistogramPercentileFunction.java
@@ -43,6 +43,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.metrics.MultiIntValuesHolder;
import org.apache.skywalking.oap.server.core.query.type.Bucket;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -74,7 +75,8 @@ public abstract class AvgHistogramPercentileFunction extends Meter implements Ac
@Setter
@Getter
- @Column(columnName = ENTITY_ID, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java
index 8721a77cda..7a44ee569d 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgLabeledFunction.java
@@ -34,6 +34,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.analysis.metrics.LabeledValueHolder;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -48,7 +49,8 @@ public abstract class AvgLabeledFunction extends Meter implements AcceptableValu
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
/**
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/latest/LatestFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/latest/LatestFunction.java
index 354988e646..c791422ab5 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/latest/LatestFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/latest/LatestFunction.java
@@ -35,6 +35,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Entranc
import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.SourceFrom;
import org.apache.skywalking.oap.server.core.query.sql.Function;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -47,7 +48,8 @@ public abstract class LatestFunction extends Meter implements AcceptableValue<Lo
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
/**
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumFunction.java
index 0649939476..c97b92f51c 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumFunction.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/sum/SumFunction.java
@@ -35,6 +35,7 @@ import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Entranc
import org.apache.skywalking.oap.server.core.analysis.metrics.annotation.SourceFrom;
import org.apache.skywalking.oap.server.core.query.sql.Function;
import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -47,7 +48,8 @@ public abstract class SumFunction extends Meter implements AcceptableValue<Long>
@Setter
@Getter
- @Column(columnName = ENTITY_ID, length = 512, shardingKeyIdx = 0)
+ @Column(columnName = ENTITY_ID, length = 512)
+ @BanyanDBShardingKey(index = 0)
private String entityId;
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java
index df4e49b435..df12484b0b 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/browser/manual/errorlog/BrowserErrorLogRecord.java
@@ -23,6 +23,7 @@ import org.apache.skywalking.oap.server.core.analysis.Stream;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
@@ -54,7 +55,8 @@ public class BrowserErrorLogRecord extends Record {
@Setter
@Getter
- @Column(columnName = SERVICE_ID, shardingKeyIdx = 0)
+ @Column(columnName = SERVICE_ID)
+ @BanyanDBShardingKey(index = 0)
private String serviceId;
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingDataRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingDataRecord.java
index dc2433bd7f..98b1519cef 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingDataRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingDataRecord.java
@@ -24,6 +24,7 @@ import lombok.Data;
import org.apache.skywalking.oap.server.core.analysis.Stream;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
@@ -48,7 +49,8 @@ public class EBPFProfilingDataRecord extends Record {
public static final String STACKS_BINARY = "dump_binary";
public static final String UPLOAD_TIME = "upload_time";
- @Column(columnName = TASK_ID, length = 600, shardingKeyIdx = 0)
+ @Column(columnName = TASK_ID, length = 600)
+ @BanyanDBShardingKey(index = 0)
private String taskId;
@Column(columnName = SCHEDULE_ID, length = 600)
private String scheduleId;
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBGlobalIndex.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBGlobalIndex.java
new file mode 100644
index 0000000000..a8dc1b761f
--- /dev/null
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBGlobalIndex.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.storage.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * BanyanDBGlobalIndex declares advanced global index, which are only available in BanyanDB.
+ *
+ * Global index should only be considered if a column value has a huge value candidates, but we will need a direct equal
+ * query without timestamp.
+ * The typical global index is designed for huge candidate of indexed values,
+ * such as `trace ID` or `segment ID + span ID`
+ *
+ * Only work with {@link Column}
+ */
+@Target({ElementType.FIELD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface BanyanDBGlobalIndex {
+ /**
+ * The current column should be indexed through global index.
+ *
+ * @return empty array if only the current column should be indexed in global index. Or list of column names if this
+ * global index includes multiple columns.
+ */
+ String[] extraFields();
+}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBShardingKey.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBShardingKey.java
new file mode 100644
index 0000000000..8c8e21a172
--- /dev/null
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/BanyanDBShardingKey.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.storage.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Sharding key is used to group time series data per metric of one entity in one place (same sharding and/or same
+ * row for column-oriented database).
+ * For example,
+ * ServiceA's traffic gauge, service call per minute, includes following timestamp values, then it should be sharded
+ * by service ID
+ * [ServiceA(encoded ID): 01-28 18:30 values-1, 01-28 18:31 values-2, 01-28 18:32 values-3, 01-28 18:32 values-4]
+ *
+ * BanyanDB is the 1st storage implementation supporting this. It would make continuous time series metrics stored
+ * closely and compressed better.
+ *
+ * 1. One entity could have multiple sharding keys
+ * 2. If no column is appointed for this, {@link org.apache.skywalking.oap.server.core.storage.StorageData#id}
+ * would be used by the storage implementation accordingly.
+ *
+ * NOTICE, this sharding concept is NOT just for splitting data into different database instances or physical
+ * files.
+ *
+ * Only work with {@link Column}
+ *
+ * @return non-negative if this column be used for sharding. -1 means not as a sharding key
+ * @since 9.1.0 created as a new annotation.
+ * @since 9.0.0 added in {@link Column}
+ */
+@Target({ElementType.FIELD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface BanyanDBShardingKey {
+ /**
+ * Relative entity tag
+ *
+ * @return index, from zero.
+ */
+ int index() default -1;
+}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java
index 077fdfe6dd..44d2305024 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java
@@ -48,11 +48,6 @@ public @interface Column {
*/
int defaultValue() default 0;
- /**
- * Match query means using analyzer(if storage have) to do key word match query.
- */
- boolean matchQuery() default false;
-
/**
* The column is just saved, never used in query.
*/
@@ -94,57 +89,6 @@ public @interface Column {
*/
ValueDataType dataType() default ValueDataType.NOT_VALUE;
- /**
- * The storage analyzer mode.
- *
- * @since 8.4.0
- */
- AnalyzerType analyzer() default AnalyzerType.OAP_ANALYZER;
-
- /**
- * Sharding key is used to group time series data per metric of one entity in one place (same sharding and/or same
- * row for column-oriented database).
- * For example,
- * ServiceA's traffic gauge, service call per minute, includes following timestamp values, then it should be sharded
- * by service ID
- * [ServiceA(encoded ID): 01-28 18:30 values-1, 01-28 18:31 values-2, 01-28 18:32 values-3, 01-28 18:32 values-4]
- *
- * BanyanDB is the 1st storage implementation supporting this. It would make continuous time series metrics stored
- * closely and compressed better.
- *
- * 1. One entity could have multiple sharding keys
- * 2. If no column is appointed for this, {@link org.apache.skywalking.oap.server.core.storage.StorageData#id}
- * would be used by the storage implementation accordingly.
- *
- * NOTICE, this sharding concept is NOT just for splitting data into different database instances or physical
- * files.
- *
- * @return non-negative if this column be used for sharding. -1 means not as a sharding key
- * @since 9.0.0
- */
- int shardingKeyIdx() default -1;
-
- /**
- * The analyzer declares the text analysis mode.
- */
- enum AnalyzerType {
- /**
- * The default analyzer.
- */
- OAP_ANALYZER("oap_analyzer"),
- /**
- * The log analyzer.
- */
- OAP_LOG_ANALYZER("oap_log_analyzer");
-
- @Getter
- private final String name;
-
- AnalyzerType(final String name) {
- this.name = name;
- }
- }
-
/**
* ValueDataType represents the data structure of value column. The persistent way of the value column determine the
* available ways to query the data.
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/ElasticSearchMatchQuery.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/ElasticSearchMatchQuery.java
new file mode 100644
index 0000000000..19084754de
--- /dev/null
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/ElasticSearchMatchQuery.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.storage.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import lombok.Getter;
+
+/**
+ * Match query is designed for ElasticSearch match query with specific analyzer. It is a fuzzy query implementation
+ * powered by analyzer.
+ *
+ * @since 9.1.0 This used to be {@link Column}'s matchQuery and analyzer attributes.
+ */
+@Target({ElementType.FIELD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface ElasticSearchMatchQuery {
+ /**
+ * The storage analyzer mode.
+ *
+ * @since 9.1.0 created as a new annotation.
+ * @since 8.4.0 added in {@link Column}
+ */
+ AnalyzerType analyzer() default AnalyzerType.OAP_ANALYZER;
+
+ /**
+ * The analyzer declares the text analysis mode.
+ */
+ enum AnalyzerType {
+ /**
+ * The default analyzer.
+ */
+ OAP_ANALYZER("oap_analyzer"),
+ /**
+ * The log analyzer.
+ */
+ OAP_LOG_ANALYZER("oap_log_analyzer");
+
+ @Getter
+ private final String name;
+
+ AnalyzerType(final String name) {
+ this.name = name;
+ }
+ }
+}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java
new file mode 100644
index 0000000000..2a8c82ee26
--- /dev/null
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/BanyanDBExtension.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.storage.model;
+
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+/**
+ * BanyanDBExtension represents extra metadata for columns, but specific for BanyanDB usages.
+ *
+ * @since 9.1.0
+ */
+@Getter
+@RequiredArgsConstructor
+public class BanyanDBExtension {
+ /**
+ * Sharding key is used to group time series data per metric of one entity. See {@link
+ * org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey#index()}.
+ *
+ * @since 9.1.0 moved into BanyanDBExtension
+ * @since 9.0.0 added into {@link ModelColumn}
+ */
+ private final int shardingKeyIdx;
+
+ /**
+ * Global index column values has 3 conditions
+ * 1. NULL, this column should be as a global index.
+ * 2. Empty array(declared by @BanyanDBGlobalIndex(extraFields = {}) in codes) represents this single column should
+ * be a global index.
+ * 3. Not empty array(declared by @BanyanDBGlobalIndex(extraFields = {"col1", "col2"}) in codes) represents this
+ * column and other declared columns should be as a global index. The values of these columns should be joint by
+ * underline(_)
+ *
+ * @since 9.1.0
+ */
+ private final String[] globalIndexColumns;
+
+ /**
+ * @return true if this column is a part of sharding key
+ */
+ public boolean isShardingKey() {
+ return this.shardingKeyIdx > -1;
+ }
+
+ /**
+ * @return null or array, see {@link #globalIndexColumns} for more details.
+ */
+ public boolean isGlobalIndexing() {
+ return globalIndexColumns != null;
+ }
+}
diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndexTest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ElasticSearchExtension.java
similarity index 55%
copy from oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndexTest.java
copy to oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ElasticSearchExtension.java
index 7331ac0ae4..4ec91d990b 100644
--- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndexTest.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ElasticSearchExtension.java
@@ -18,21 +18,25 @@
package org.apache.skywalking.oap.server.core.storage.model;
-import org.junit.Assert;
-import org.junit.Test;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
-public class ExtraQueryIndexTest {
- @Test
- public void testIndexColumns() {
- final ExtraQueryIndex extraQueryIndex = new ExtraQueryIndex("a1", new String[] {"a2"});
- Assert.assertArrayEquals(new String[] {
- "a1",
- "a2"
- }, extraQueryIndex.getColumns());
- }
+/**
+ * ElasticSearchExtension represents extra metadata for columns, but specific for ElasticSearch usages.
+ *
+ * @since 9.1.0
+ */
+@Getter
+@RequiredArgsConstructor
+public class ElasticSearchExtension {
+ /**
+ * The analyzer policy appointed to fuzzy query, especially for ElasticSearch.
+ * When it is null, it means no need to build match query, no `copy_to` column, and no analyzer assigned.
+ */
+ private final ElasticSearchMatchQuery.AnalyzerType analyzer;
- @Test(expected = IllegalArgumentException.class)
- public void testIllegalIndexColumns() {
- ExtraQueryIndex extraQueryIndex = new ExtraQueryIndex("a1", new String[0]);
+ public boolean needMatchQuery() {
+ return analyzer != null;
}
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndex.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndex.java
deleted file mode 100644
index d5f8fa8b9d..0000000000
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndex.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.skywalking.oap.server.core.storage.model;
-
-import lombok.Getter;
-import org.apache.skywalking.oap.server.library.util.CollectionUtils;
-
-/**
- * The extra query index if the storage could support this mode. Many NO-SQL support one column index only, in that
- * case, this could be ignored in the implementation level.
- */
-@Getter
-public class ExtraQueryIndex {
- private String[] columns;
-
- public ExtraQueryIndex(String mainColumn, final String[] withColumns) {
- if (CollectionUtils.isNotEmpty(withColumns)) {
- columns = new String[withColumns.length + 1];
- columns[0] = mainColumn;
- System.arraycopy(withColumns, 0, columns, 1, withColumns.length);
- } else {
- throw new IllegalArgumentException("ExtraQueryIndex required withColumns as a not empty list.");
- }
-
- }
-
- /**
- * Keep the same name replacement as {@link ColumnName#overrideName(String, String)}
- *
- * @param oldName to be replaced.
- * @param newName to use in the storage level.
- */
- public void overrideName(String oldName, String newName) {
- for (int i = 0; i < columns.length; i++) {
- if (columns[i].equals(oldName)) {
- columns[i] = newName;
- }
- }
- }
-}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java
index b37375688f..1550e40ccf 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/Model.java
@@ -31,7 +31,6 @@ import org.apache.skywalking.oap.server.core.analysis.DownSampling;
public class Model {
private final String name;
private final List<ModelColumn> columns;
- private final List<ExtraQueryIndex> extraQueryIndices;
private final int scopeId;
private final DownSampling downsampling;
private final boolean record;
@@ -42,7 +41,6 @@ public class Model {
public Model(final String name,
final List<ModelColumn> columns,
- final List<ExtraQueryIndex> extraQueryIndices,
final int scopeId,
final DownSampling downsampling,
final boolean record,
@@ -51,7 +49,6 @@ public class Model {
boolean timeRelativeID) {
this.name = name;
this.columns = columns;
- this.extraQueryIndices = extraQueryIndices;
this.scopeId = scopeId;
this.downsampling = downsampling;
this.isTimeSeries = !DownSampling.None.equals(downsampling);
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java
index 9d381f19ae..b5dbfbe739 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java
@@ -22,7 +22,6 @@ import java.lang.reflect.Type;
import lombok.Getter;
import lombok.ToString;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
-import org.apache.skywalking.oap.server.core.storage.annotation.Column;
@Getter
@ToString
@@ -30,7 +29,6 @@ public class ModelColumn {
private final ColumnName columnName;
private final Class<?> type;
private final Type genericType;
- private final boolean matchQuery;
/**
* Storage this column for query result, but can't be as a condition . Conflict with {@link #indexOnly}
*/
@@ -45,33 +43,42 @@ public class ModelColumn {
* The max length of column value for length sensitive database.
*/
private final int length;
+
/**
- * The analyzer policy appointed to fuzzy query, especially for ElasticSearch
+ * Hold configurations especially for SQL Database, such as MySQL, H2, PostgreSQL
+ *
+ * @since 9.1.0
*/
- private final Column.AnalyzerType analyzer;
+ private SQLDatabaseExtension sqlDatabaseExtension;
/**
- * Sharding key is used to group time series data per metric of one entity. See {@link Column#shardingKeyIdx()}.
+ * Hold configurations especially for ElasticSearch
*
- * @since 9.0.0
+ * @since 9.1.0
+ */
+ private ElasticSearchExtension elasticSearchExtension;
+ /**
+ * Hold configurations especially for BanyanDB relevant
+ *
+ * @since 9.1.0
*/
- private int shardingKeyIdx;
+ private BanyanDBExtension banyanDBExtension;
public ModelColumn(ColumnName columnName,
Class<?> type,
Type genericType,
- boolean matchQuery,
boolean storageOnly,
boolean indexOnly,
boolean isValue,
int length,
- Column.AnalyzerType analyzer,
- int shardingKeyIdx) {
+ SQLDatabaseExtension sqlDatabaseExtension,
+ ElasticSearchExtension elasticSearchExtension,
+ BanyanDBExtension banyanDBExtension) {
this.columnName = columnName;
this.type = type;
this.genericType = genericType;
- this.matchQuery = matchQuery;
this.length = length;
- this.analyzer = analyzer;
+ this.sqlDatabaseExtension = sqlDatabaseExtension;
+ this.elasticSearchExtension = elasticSearchExtension;
/*
* byte[] and {@link IntKeyLongValueHashMap} could never be query.
*/
@@ -90,6 +97,6 @@ public class ModelColumn {
"The column " + columnName + " can't be defined as both indexOnly and storageOnly.");
}
this.indexOnly = indexOnly;
- this.shardingKeyIdx = shardingKeyIdx;
+ this.banyanDBExtension = banyanDBExtension;
}
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/SQLDatabaseExtension.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/SQLDatabaseExtension.java
new file mode 100644
index 0000000000..c7df66d4d5
--- /dev/null
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/SQLDatabaseExtension.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.storage.model;
+
+import java.util.ArrayList;
+import java.util.List;
+import lombok.Getter;
+import org.apache.skywalking.oap.server.library.util.CollectionUtils;
+
+/**
+ * @since 9.0.0
+ */
+@Getter
+public class SQLDatabaseExtension {
+ private final List<MultiColumnsIndex> indices = new ArrayList<>(5);
+
+ public void appendIndex(MultiColumnsIndex index) {
+ indices.add(index);
+ }
+
+ /**
+ * The multiple columns index if the storage could support this mode. Many NO-SQL support one column index only, in that
+ * case, this could be ignored in the implementation level.
+ */
+ @Getter
+ public static class MultiColumnsIndex {
+ private String[] columns;
+
+ public MultiColumnsIndex(String mainColumn, final String[] withColumns) {
+ if (CollectionUtils.isNotEmpty(withColumns)) {
+ columns = new String[withColumns.length + 1];
+ columns[0] = mainColumn;
+ System.arraycopy(withColumns, 0, columns, 1, withColumns.length);
+ } else {
+ throw new IllegalArgumentException("ExtraQueryIndex required withColumns as a not empty list.");
+ }
+
+ }
+
+ /**
+ * Keep the same name replacement as {@link ColumnName#overrideName(String, String)}
+ *
+ * @param oldName to be replaced.
+ * @param newName to use in the storage level.
+ */
+ public void overrideName(String oldName, String newName) {
+ for (int i = 0; i < columns.length; i++) {
+ if (columns[i].equals(oldName)) {
+ columns[i] = newName;
+ }
+ }
+ }
+ }
+}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
index 8aebfb1dc4..e71aef4408 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
@@ -27,7 +27,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.core.analysis.FunctionCategory;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.StorageException;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBGlobalIndex;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.core.storage.annotation.MultipleQueryUnifiedIndex;
import org.apache.skywalking.oap.server.core.storage.annotation.QueryUnifiedIndex;
import org.apache.skywalking.oap.server.core.storage.annotation.Storage;
@@ -56,14 +59,16 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
DefaultScopeDefine.nameOf(scopeId);
List<ModelColumn> modelColumns = new ArrayList<>();
- List<ExtraQueryIndex> extraQueryIndices = new ArrayList<>();
ShardingKeyChecker checker = new ShardingKeyChecker();
- retrieval(aClass, storage.getModelName(), modelColumns, extraQueryIndices, scopeId, checker);
+ retrieval(aClass, storage.getModelName(), modelColumns, scopeId, checker);
checker.check(storage.getModelName());
Model model = new Model(
- storage.getModelName(), modelColumns, extraQueryIndices, scopeId,
- storage.getDownsampling(), record,
+ storage.getModelName(),
+ modelColumns,
+ scopeId,
+ storage.getDownsampling(),
+ record,
isSuperDatasetModel(aClass),
FunctionCategory.uniqueFunctionName(aClass),
storage.isTimeRelativeID()
@@ -100,7 +105,6 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
private void retrieval(final Class<?> clazz,
final String modelName,
final List<ModelColumn> modelColumns,
- final List<ExtraQueryIndex> extraQueryIndices,
final int scopeId,
ShardingKeyChecker checker) {
if (log.isDebugEnabled()) {
@@ -129,13 +133,55 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
}
}
+ // SQL Database extension
+ SQLDatabaseExtension sqlDatabaseExtension = new SQLDatabaseExtension();
+ List<QueryUnifiedIndex> indexDefinitions = new ArrayList<>();
+ if (field.isAnnotationPresent(QueryUnifiedIndex.class)) {
+ indexDefinitions.add(field.getAnnotation(QueryUnifiedIndex.class));
+ }
+
+ if (field.isAnnotationPresent(MultipleQueryUnifiedIndex.class)) {
+ Collections.addAll(indexDefinitions, field.getAnnotation(MultipleQueryUnifiedIndex.class).value());
+ }
+
+ indexDefinitions.forEach(indexDefinition -> {
+ sqlDatabaseExtension.appendIndex(new SQLDatabaseExtension.MultiColumnsIndex(
+ column.columnName(),
+ indexDefinition.withColumns()
+ ));
+ });
+
+ // ElasticSearch extension
+ final ElasticSearchMatchQuery elasticSearchAnalyzer = field.getAnnotation(
+ ElasticSearchMatchQuery.class);
+ ElasticSearchExtension elasticSearchExtension = new ElasticSearchExtension(
+ elasticSearchAnalyzer == null ? null : elasticSearchAnalyzer.analyzer()
+ );
+
+ // BanyanDB extension
+ final BanyanDBShardingKey banyanDBShardingKey = field.getAnnotation(BanyanDBShardingKey.class);
+ final BanyanDBGlobalIndex banyanDBGlobalIndex = field.getAnnotation(BanyanDBGlobalIndex.class);
+ BanyanDBExtension banyanDBExtension = new BanyanDBExtension(
+ banyanDBShardingKey == null ? -1 : banyanDBShardingKey.index(),
+ banyanDBGlobalIndex == null ? null : banyanDBGlobalIndex.extraFields()
+ );
+
final ModelColumn modelColumn = new ModelColumn(
- new ColumnName(modelName, column.columnName()), field.getType(), field.getGenericType(),
- column.matchQuery(), column.storageOnly(), column.indexOnly(), column.dataType().isValue(),
+ new ColumnName(
+ modelName,
+ column.columnName()
+ ),
+ field.getType(),
+ field.getGenericType(),
+ column.storageOnly(),
+ column.indexOnly(),
+ column.dataType().isValue(),
columnLength,
- column.analyzer(), column.shardingKeyIdx()
+ sqlDatabaseExtension,
+ elasticSearchExtension,
+ banyanDBExtension
);
- if (column.shardingKeyIdx() > -1) {
+ if (banyanDBExtension.isShardingKey()) {
checker.accept(modelName, modelColumn);
}
@@ -149,25 +195,11 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
column.defaultValue(), scopeId
);
}
-
- List<QueryUnifiedIndex> indexDefinitions = new ArrayList<>();
- if (field.isAnnotationPresent(QueryUnifiedIndex.class)) {
- indexDefinitions.add(field.getAnnotation(QueryUnifiedIndex.class));
- }
-
- if (field.isAnnotationPresent(MultipleQueryUnifiedIndex.class)) {
- Collections.addAll(indexDefinitions, field.getAnnotation(MultipleQueryUnifiedIndex.class).value());
- }
-
- indexDefinitions.forEach(indexDefinition -> extraQueryIndices.add(new ExtraQueryIndex(
- column.columnName(),
- indexDefinition.withColumns()
- )));
}
}
if (Objects.nonNull(clazz.getSuperclass())) {
- retrieval(clazz.getSuperclass(), modelName, modelColumns, extraQueryIndices, scopeId, checker);
+ retrieval(clazz.getSuperclass(), modelName, modelColumns, scopeId, checker);
}
}
@@ -180,8 +212,12 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
private void followColumnNameRules(Model model) {
columnNameOverrideRule.forEach((oldName, newName) -> {
- model.getColumns().forEach(column -> column.getColumnName().overrideName(oldName, newName));
- model.getExtraQueryIndices().forEach(extraQueryIndex -> extraQueryIndex.overrideName(oldName, newName));
+ model.getColumns().forEach(column -> {
+ column.getColumnName().overrideName(oldName, newName);
+ column.getSqlDatabaseExtension()
+ .getIndices()
+ .forEach(extraQueryIndex -> extraQueryIndex.overrideName(oldName, newName));
+ });
});
}
@@ -197,7 +233,7 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
* @throws IllegalStateException if sharding key indices are conflicting.
*/
private void accept(String modelName, ModelColumn modelColumn) throws IllegalStateException {
- final int idx = modelColumn.getShardingKeyIdx();
+ final int idx = modelColumn.getBanyanDBExtension().getShardingKeyIdx();
while (idx + 1 > keys.size()) {
keys.add(null);
}
@@ -206,7 +242,8 @@ public class StorageModels implements IModelManager, ModelCreator, ModelManipula
throw new IllegalStateException(
modelName + "'s "
+ "Column [" + exist.getColumnName() + "] and column [" + modelColumn.getColumnName()
- + " are conflicting with sharding key index=" + modelColumn.getShardingKeyIdx());
+ + " are conflicting with sharding key index=" + modelColumn.getBanyanDBExtension()
+ .getShardingKeyIdx());
}
keys.set(idx, modelColumn);
}
diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java
index 90591834b3..be61edb872 100644
--- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java
+++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java
@@ -19,31 +19,38 @@
package org.apache.skywalking.oap.server.core.storage.model;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
-import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.junit.Assert;
import org.junit.Test;
public class ModelColumnTest {
@Test
public void testColumnDefine() {
- ModelColumn column = new ModelColumn(new ColumnName("", "abc"), byte[].class, byte[].class, true,
+ ModelColumn column = new ModelColumn(new ColumnName("", "abc"), byte[].class, byte[].class,
false, false, true, 0,
- Column.AnalyzerType.OAP_ANALYZER, 0
+ new SQLDatabaseExtension(),
+ new ElasticSearchExtension(
+ ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER),
+ new BanyanDBExtension(-1, null)
);
Assert.assertEquals(true, column.isStorageOnly());
Assert.assertEquals("abc", column.getColumnName().getName());
- column = new ModelColumn(new ColumnName("", "abc"), DataTable.class, DataTable.class, true,
+ column = new ModelColumn(new ColumnName("", "abc"), DataTable.class, DataTable.class,
false, false, true, 200,
- Column.AnalyzerType.OAP_ANALYZER, 0
+ new SQLDatabaseExtension(),
+ new ElasticSearchExtension(ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER),
+ new BanyanDBExtension(-1, null)
);
Assert.assertEquals(true, column.isStorageOnly());
Assert.assertEquals("abc", column.getColumnName().getName());
Assert.assertEquals(200, column.getLength());
- column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class, true,
+ column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class,
false, false, true, 200,
- Column.AnalyzerType.OAP_ANALYZER, 0
+ new SQLDatabaseExtension(),
+ new ElasticSearchExtension(ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER),
+ new BanyanDBExtension(-1, null)
);
Assert.assertEquals(false, column.isStorageOnly());
Assert.assertEquals("abc", column.getColumnName().getName());
@@ -52,16 +59,22 @@ public class ModelColumnTest {
@Test(expected = IllegalArgumentException.class)
public void testConflictDefinition() {
ModelColumn column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class,
- true, true, false, true, 200,
- Column.AnalyzerType.OAP_ANALYZER, 0
+ true, false, true, 200,
+ new SQLDatabaseExtension(),
+ new ElasticSearchExtension(
+ ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER),
+ new BanyanDBExtension(-1, null)
);
}
@Test(expected = IllegalArgumentException.class)
public void testConflictDefinitionIndexOnly() {
ModelColumn column = new ModelColumn(new ColumnName("", "abc"), String.class, String.class,
- true, true, true, false, 200,
- Column.AnalyzerType.OAP_ANALYZER, 0
+ true, true, false, 200,
+ new SQLDatabaseExtension(),
+ new ElasticSearchExtension(
+ ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER),
+ new BanyanDBExtension(-1, null)
);
}
}
diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndexTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/SQLDatabaseExtensionTest.java
similarity index 78%
rename from oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndexTest.java
rename to oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/SQLDatabaseExtensionTest.java
index 7331ac0ae4..1f789a4894 100644
--- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ExtraQueryIndexTest.java
+++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/SQLDatabaseExtensionTest.java
@@ -21,10 +21,11 @@ package org.apache.skywalking.oap.server.core.storage.model;
import org.junit.Assert;
import org.junit.Test;
-public class ExtraQueryIndexTest {
+public class SQLDatabaseExtensionTest {
@Test
public void testIndexColumns() {
- final ExtraQueryIndex extraQueryIndex = new ExtraQueryIndex("a1", new String[] {"a2"});
+ final SQLDatabaseExtension.MultiColumnsIndex extraQueryIndex = new SQLDatabaseExtension.MultiColumnsIndex(
+ "a1", new String[] {"a2"});
Assert.assertArrayEquals(new String[] {
"a1",
"a2"
@@ -33,6 +34,7 @@ public class ExtraQueryIndexTest {
@Test(expected = IllegalArgumentException.class)
public void testIllegalIndexColumns() {
- ExtraQueryIndex extraQueryIndex = new ExtraQueryIndex("a1", new String[0]);
+ SQLDatabaseExtension.MultiColumnsIndex extraQueryIndex = new SQLDatabaseExtension.MultiColumnsIndex(
+ "a1", new String[0]);
}
}
diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/StorageModelsTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/StorageModelsTest.java
index 961c015f23..65d734ef47 100644
--- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/StorageModelsTest.java
+++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/StorageModelsTest.java
@@ -74,12 +74,10 @@ public class StorageModelsTest {
Assert.assertEquals(false, model.getColumns().get(2).isStorageOnly());
Assert.assertEquals(true, model.getColumns().get(3).isStorageOnly());
- final List<ExtraQueryIndex> extraQueryIndices = model.getExtraQueryIndices();
- Assert.assertEquals(3, extraQueryIndices.size());
Assert.assertArrayEquals(new String[] {
"column2",
"column"
- }, extraQueryIndices.get(2).getColumns());
+ }, model.getColumns().get(2).getSqlDatabaseExtension().getIndices().get(1).getColumns());
}
@Stream(name = "StorageModelsTest", scopeId = -1, builder = TestModel.Builder.class, processor = MetricsStreamProcessor.class)
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/AnalyzerSetting.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/AnalyzerSetting.java
index 1de991027b..57e79ad0a4 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/AnalyzerSetting.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/AnalyzerSetting.java
@@ -27,7 +27,7 @@ import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.core.storage.StorageException;
-import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearchMatchQuery;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.StorageModuleElasticsearchConfig;
@Getter
@@ -79,18 +79,18 @@ public class AnalyzerSetting {
public enum Generator {
OAP_ANALYZER_SETTING_GENERATOR(
- Column.AnalyzerType.OAP_ANALYZER,
+ ElasticSearchMatchQuery.AnalyzerType.OAP_ANALYZER,
config -> new Gson().fromJson(config.getOapAnalyzer(), AnalyzerSetting.class)
),
OAP_LOG_ANALYZER_SETTING_GENERATOR(
- Column.AnalyzerType.OAP_LOG_ANALYZER,
+ ElasticSearchMatchQuery.AnalyzerType.OAP_LOG_ANALYZER,
config -> new Gson().fromJson(config.getOapLogAnalyzer(), AnalyzerSetting.class)
);
- private final Column.AnalyzerType type;
+ private final ElasticSearchMatchQuery.AnalyzerType type;
private final GenerateAnalyzerSettingFunc func;
- Generator(final Column.AnalyzerType type,
+ Generator(final ElasticSearchMatchQuery.AnalyzerType type,
final GenerateAnalyzerSettingFunc func) {
this.type = type;
this.func = func;
@@ -100,7 +100,7 @@ public class AnalyzerSetting {
return this.func;
}
- public static Generator getGenerator(Column.AnalyzerType type) throws StorageException {
+ public static Generator getGenerator(ElasticSearchMatchQuery.AnalyzerType type) throws StorageException {
for (final Generator value : Generator.values()) {
if (value.type == type) {
return value;
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
index e77c666adb..3e59f249ec 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
-import org.apache.skywalking.oap.server.library.util.StringUtil;
import org.apache.skywalking.library.elasticsearch.response.Index;
import org.apache.skywalking.library.elasticsearch.response.IndexTemplate;
import org.apache.skywalking.library.elasticsearch.response.Mappings;
@@ -36,6 +35,7 @@ import org.apache.skywalking.oap.server.core.storage.model.ModelInstaller;
import org.apache.skywalking.oap.server.library.client.Client;
import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
+import org.apache.skywalking.oap.server.library.util.StringUtil;
import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.StorageModuleElasticsearchConfig;
@Slf4j
@@ -214,7 +214,11 @@ public class StorageEsInstaller extends ModelInstaller {
private Map getAnalyzerSetting(List<ModelColumn> analyzerTypes) throws StorageException {
AnalyzerSetting analyzerSetting = new AnalyzerSetting();
for (final ModelColumn column : analyzerTypes) {
- AnalyzerSetting setting = AnalyzerSetting.Generator.getGenerator(column.getAnalyzer())
+ if (!column.getElasticSearchExtension().needMatchQuery()) {
+ continue;
+ }
+ AnalyzerSetting setting = AnalyzerSetting.Generator.getGenerator(
+ column.getElasticSearchExtension().getAnalyzer())
.getGenerateFunc()
.generate(config);
analyzerSetting.combine(setting);
@@ -227,7 +231,7 @@ public class StorageEsInstaller extends ModelInstaller {
Mappings.Source source = new Mappings.Source();
for (ModelColumn columnDefine : model.getColumns()) {
final String type = columnTypeEsMapping.transform(columnDefine.getType(), columnDefine.getGenericType());
- if (columnDefine.isMatchQuery()) {
+ if (columnDefine.getElasticSearchExtension().needMatchQuery()) {
String matchCName = MatchCNameBuilder.INSTANCE.build(columnDefine.getColumnName().getName());
Map<String, Object> originalColumn = new HashMap<>();
@@ -237,7 +241,7 @@ public class StorageEsInstaller extends ModelInstaller {
Map<String, Object> matchColumn = new HashMap<>();
matchColumn.put("type", "text");
- matchColumn.put("analyzer", columnDefine.getAnalyzer().getName());
+ matchColumn.put("analyzer", columnDefine.getElasticSearchExtension().getAnalyzer().getName());
properties.put(matchCName, matchColumn);
} else {
Map<String, Object> column = new HashMap<>();
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java
index 9d6a3d3869..369275d74c 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/TimeSeriesUtilsTest.java
@@ -36,13 +36,13 @@ public class TimeSeriesUtilsTest {
@Before
public void prepare() {
- superDatasetModel = new Model("superDatasetModel", Lists.newArrayList(), Lists.newArrayList(),
+ superDatasetModel = new Model("superDatasetModel", Lists.newArrayList(),
0, DownSampling.Minute, true, true, "", true
);
- normalRecordModel = new Model("normalRecordModel", Lists.newArrayList(), Lists.newArrayList(),
+ normalRecordModel = new Model("normalRecordModel", Lists.newArrayList(),
0, DownSampling.Minute, true, false, "", true
);
- normalMetricsModel = new Model("normalMetricsModel", Lists.newArrayList(), Lists.newArrayList(),
+ normalMetricsModel = new Model("normalMetricsModel", Lists.newArrayList(),
0, DownSampling.Minute, false, false, "", true
);
TimeSeriesUtils.setSUPER_DATASET_DAY_STEP(1);
diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java
index 3efd6890b3..cd4b4da0d7 100644
--- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java
+++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java
@@ -25,9 +25,9 @@ import java.sql.SQLException;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.core.storage.StorageException;
-import org.apache.skywalking.oap.server.core.storage.model.ExtraQueryIndex;
import org.apache.skywalking.oap.server.core.storage.model.Model;
import org.apache.skywalking.oap.server.core.storage.model.ModelColumn;
+import org.apache.skywalking.oap.server.core.storage.model.SQLDatabaseExtension;
import org.apache.skywalking.oap.server.core.storage.type.StorageDataComplexObject;
import org.apache.skywalking.oap.server.library.client.Client;
import org.apache.skywalking.oap.server.library.client.jdbc.JDBCClientException;
@@ -104,22 +104,25 @@ public class MySQLTableInstaller extends H2TableInstaller {
}
}
- for (final ExtraQueryIndex extraQueryIndex : model.getExtraQueryIndices()) {
- SQLBuilder tableIndexSQL = new SQLBuilder("CREATE INDEX ");
- tableIndexSQL.append(model.getName().toUpperCase())
- .append("_")
- .append(String.valueOf(indexSeq++))
- .append("_IDX ");
- tableIndexSQL.append(" ON ").append(model.getName()).append("(");
- final String[] columns = extraQueryIndex.getColumns();
- for (int i = 0; i < columns.length; i++) {
- tableIndexSQL.append(columns[i]);
- if (i < columns.length - 1) {
- tableIndexSQL.append(",");
+ for (final ModelColumn modelColumn : model.getColumns()) {
+ for (final SQLDatabaseExtension.MultiColumnsIndex index : modelColumn.getSqlDatabaseExtension()
+ .getIndices()) {
+ SQLBuilder tableIndexSQL = new SQLBuilder("CREATE INDEX ");
+ tableIndexSQL.append(model.getName().toUpperCase())
+ .append("_")
+ .append(String.valueOf(indexSeq++))
+ .append("_IDX ");
+ tableIndexSQL.append(" ON ").append(model.getName()).append("(");
+ final String[] columns = index.getColumns();
+ for (int i = 0; i < columns.length; i++) {
+ tableIndexSQL.append(columns[i]);
+ if (i < columns.length - 1) {
+ tableIndexSQL.append(",");
+ }
}
+ tableIndexSQL.append(")");
+ createIndex(client, connection, model, tableIndexSQL);
}
- tableIndexSQL.append(")");
- createIndex(client, connection, model, tableIndexSQL);
}
}
diff --git a/oap-server/server-storage-plugin/storage-zipkin-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java b/oap-server/server-storage-plugin/storage-zipkin-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java
index ea4bbb13e4..16376804f8 100644
--- a/oap-server/server-storage-plugin/storage-zipkin-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java
+++ b/oap-server/server-storage-plugin/storage-zipkin-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/ZipkinSpanRecord.java
@@ -25,6 +25,7 @@ import org.apache.skywalking.oap.server.core.analysis.Stream;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDBShardingKey;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.SuperDataset;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
@@ -60,15 +61,17 @@ public class ZipkinSpanRecord extends Record {
private String spanId;
@Setter
@Getter
- @Column(columnName = SERVICE_ID, shardingKeyIdx = 0)
+ @Column(columnName = SERVICE_ID)
+ @BanyanDBShardingKey(index = 0)
private String serviceId;
@Setter
@Getter
- @Column(columnName = SERVICE_INSTANCE_ID, shardingKeyIdx = 1)
+ @Column(columnName = SERVICE_INSTANCE_ID)
+ @BanyanDBShardingKey(index = 1)
private String serviceInstanceId;
@Setter
@Getter
- @Column(columnName = ENDPOINT_NAME, matchQuery = true)
+ @Column(columnName = ENDPOINT_NAME)
private String endpointName;
@Setter
@Getter
@@ -88,7 +91,8 @@ public class ZipkinSpanRecord extends Record {
private int latency;
@Setter
@Getter
- @Column(columnName = IS_ERROR, shardingKeyIdx = 2)
+ @Column(columnName = IS_ERROR)
+ @BanyanDBShardingKey(index = 2)
private int isError;
@Setter
@Getter