You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2020/03/29 01:58:47 UTC
[skywalking] 01/01: 1. Add length definition with reasonable
default value. 2. Merge H2 and MySQL columntype mapping back to consistent.
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch core-refactor
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 4798983c4f73e6d52b5a6f85a353c8b1776363c9
Author: Wu Sheng <wu...@foxmail.com>
AuthorDate: Sun Mar 29 09:58:07 2020 +0800
1. Add length definition with reasonable default value.
2. Merge H2 and MySQL columntype mapping back to consistent.
---
.../org/apache/skywalking/oal/rt/OALRuntime.java | 47 +++++++++++++-----
.../skywalking/oal/rt/parser/SourceColumn.java | 4 +-
.../oal/rt/parser/SourceColumnsFactory.java | 5 +-
.../EndpointRelationServerSideMetrics.java | 2 +-
.../ServiceInstanceRelationClientSideMetrics.java | 2 +-
.../ServiceInstanceRelationServerSideMetrics.java | 2 +-
.../service/ServiceRelationClientSideMetrics.java | 2 +-
.../service/ServiceRelationServerSideMetrics.java | 2 +-
.../analysis/manual/segment/SegmentRecord.java | 8 ++--
.../server/core/analysis/metrics/ApdexMetrics.java | 6 +--
.../server/core/analysis/metrics/CPMMetrics.java | 2 +-
.../core/analysis/metrics/DoubleAvgMetrics.java | 4 +-
.../core/analysis/metrics/LongAvgMetrics.java | 4 +-
.../core/analysis/metrics/PercentileMetrics.java | 10 ++--
.../server/core/analysis/metrics/PxxMetrics.java | 4 +-
.../analysis/metrics/ThermodynamicMetrics.java | 10 ++--
.../oap/server/core/analysis/topn/TopN.java | 2 +-
.../oap/server/core/source/DefaultScopeDefine.java | 8 ++--
.../oap/server/core/source/ScopeDefaultColumn.java | 18 +++++--
.../oap/server/core/storage/annotation/Column.java | 9 +++-
.../oap/server/core/storage/model/ColumnName.java | 3 ++
.../oap/server/core/storage/model/ModelColumn.java | 30 ++++++++++--
.../server/core/storage/model/StorageModels.java | 15 ++++--
.../server/core/storage/model/ModelColumnTest.java | 55 ++++++++++++++++++++++
.../elasticsearch/base/StorageEsInstaller.java | 8 ++--
.../plugin/influxdb/InfluxStorageProvider.java | 4 +-
.../plugin/jdbc/h2/dao/H2TableInstaller.java | 16 +++----
.../plugin/jdbc/mysql/MySQLTableInstaller.java | 39 ---------------
28 files changed, 206 insertions(+), 115 deletions(-)
diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java
index 33d6fa3..6a7cde1 100644
--- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java
+++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java
@@ -236,20 +236,25 @@ public class OALRuntime implements OALEngine {
*/
for (SourceColumn field : metricsStmt.getFieldsFromSource()) {
try {
- CtField newField = CtField.make("private " + field.getType()
- .getName() + " " + field.getFieldName() + ";", metricsClass);
+ CtField newField = CtField.make(
+ "private " + field.getType()
+ .getName() + " " + field.getFieldName() + ";", metricsClass);
metricsClass.addField(newField);
metricsClass.addMethod(CtNewMethod.getter(field.getFieldGetter(), newField));
metricsClass.addMethod(CtNewMethod.setter(field.getFieldSetter(), newField));
- AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(constPool, AnnotationsAttribute.visibleTag);
+ AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(
+ constPool, AnnotationsAttribute.visibleTag);
/**
* Add @Column(columnName = "${sourceField.columnName}")
*/
Annotation columnAnnotation = new Annotation(Column.class.getName(), constPool);
columnAnnotation.addMemberValue("columnName", new StringMemberValue(field.getColumnName(), constPool));
+ if (field.getType().equals(String.class)) {
+ columnAnnotation.addMemberValue("length", new IntegerMemberValue(field.getLength(), constPool));
+ }
annotationsAttribute.addAnnotation(columnAnnotation);
if (field.isID()) {
@@ -263,7 +268,8 @@ public class OALRuntime implements OALEngine {
newField.getFieldInfo().addAttribute(annotationsAttribute);
} catch (CannotCompileException e) {
- logger.error("Can't add field(including set/get) " + field.getFieldName() + " in " + className + ".", e);
+ logger.error(
+ "Can't add field(including set/get) " + field.getFieldName() + " in " + className + ".", e);
throw new OALCompileException(e.getMessage(), e);
}
}
@@ -287,11 +293,13 @@ public class OALRuntime implements OALEngine {
*
* at Stream(name = "${tableName}", scopeId = ${sourceScopeId}, builder = ${metricsName}Metrics.Builder.class, processor = MetricsStreamProcessor.class)
*/
- AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(constPool, AnnotationsAttribute.visibleTag);
+ AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(
+ constPool, AnnotationsAttribute.visibleTag);
Annotation streamAnnotation = new Annotation(Stream.class.getName(), constPool);
streamAnnotation.addMemberValue("name", new StringMemberValue(metricsStmt.getTableName(), constPool));
streamAnnotation.addMemberValue("scopeId", new IntegerMemberValue(constPool, metricsStmt.getSourceScopeId()));
- streamAnnotation.addMemberValue("builder", new ClassMemberValue(metricsBuilderClassName(metricsStmt, true), constPool));
+ streamAnnotation.addMemberValue(
+ "builder", new ClassMemberValue(metricsBuilderClassName(metricsStmt, true), constPool));
streamAnnotation.addMemberValue("processor", new ClassMemberValue(METRICS_STREAM_PROCESSOR, constPool));
annotationsAttribute.addAnnotation(streamAnnotation);
@@ -328,7 +336,8 @@ public class OALRuntime implements OALEngine {
* Create empty construct
*/
try {
- CtConstructor defaultConstructor = CtNewConstructor.make("public " + className + "() {}", metricsBuilderClass);
+ CtConstructor defaultConstructor = CtNewConstructor.make(
+ "public " + className + "() {}", metricsBuilderClass);
metricsBuilderClass.addConstructor(defaultConstructor);
} catch (CannotCompileException e) {
logger.error("Can't add empty constructor in " + className + ".", e);
@@ -363,7 +372,7 @@ public class OALRuntime implements OALEngine {
* Generate SourceDispatcher class and inject it to classloader
*/
private Class generateDispatcherClass(String scopeName,
- DispatcherContext dispatcherContext) throws OALCompileException {
+ DispatcherContext dispatcherContext) throws OALCompileException {
String className = dispatcherClassName(scopeName, false);
CtClass dispatcherClass = classPool.makeClass(dispatcherClassName(scopeName, true));
@@ -377,10 +386,19 @@ public class OALRuntime implements OALEngine {
*/
String sourceClassName = SOURCE_PACKAGE + dispatcherContext.getSource();
SignatureAttribute.ClassSignature dispatcherSignature = new SignatureAttribute.ClassSignature(null, null,
- // Set interface and its generic params
- new SignatureAttribute.ClassType[] {
- new SignatureAttribute.ClassType(SourceDispatcher.class.getCanonicalName(), new SignatureAttribute.TypeArgument[] {new SignatureAttribute.TypeArgument(new SignatureAttribute.ClassType(sourceClassName))})
- });
+ // Set interface and its generic params
+ new SignatureAttribute.ClassType[] {
+ new SignatureAttribute.ClassType(
+ SourceDispatcher.class
+ .getCanonicalName(),
+ new SignatureAttribute.TypeArgument[] {
+ new SignatureAttribute.TypeArgument(
+ new SignatureAttribute.ClassType(
+ sourceClassName))
+ }
+ )
+ }
+ );
dispatcherClass.setGenericSignature(dispatcherSignature.encode());
} catch (NotFoundException e) {
@@ -397,7 +415,10 @@ public class OALRuntime implements OALEngine {
configuration.getTemplate("dispatcher/doMetrics.ftl").process(dispatcherContextMetric, methodEntity);
dispatcherClass.addMethod(CtNewMethod.make(methodEntity.toString(), dispatcherClass));
} catch (Exception e) {
- logger.error("Can't generate method do" + dispatcherContextMetric.getMetricsName() + " for " + className + ".", e);
+ logger.error(
+ "Can't generate method do" + dispatcherContextMetric.getMetricsName() + " for " + className + ".",
+ e
+ );
logger.error("Method body as following" + System.lineSeparator() + "{}", methodEntity);
throw new OALCompileException(e.getMessage(), e);
}
diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumn.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumn.java
index 59d442a..fc798a3 100644
--- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumn.java
+++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumn.java
@@ -32,15 +32,17 @@ public class SourceColumn {
private Class<?> type;
private String typeName;
private boolean isID;
+ private int length;
private String fieldSetter;
private String fieldGetter;
- public SourceColumn(String fieldName, String columnName, Class<?> type, boolean isID) {
+ public SourceColumn(String fieldName, String columnName, Class<?> type, boolean isID, int length) {
this.fieldName = fieldName;
this.columnName = columnName;
this.type = type;
this.typeName = type.getName();
this.isID = isID;
+ this.length = length;
this.fieldGetter = ClassMethodUtil.toGetMethod(fieldName);
this.fieldSetter = ClassMethodUtil.toSetMethod(fieldName);
diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumnsFactory.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumnsFactory.java
index 12f1cac..2339513 100644
--- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumnsFactory.java
+++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumnsFactory.java
@@ -29,8 +29,9 @@ public class SourceColumnsFactory {
List<ScopeDefaultColumn> columns = DefaultScopeDefine.getDefaultColumns(source);
for (ScopeDefaultColumn defaultColumn : columns) {
- sourceColumns.add(new SourceColumn(defaultColumn.getFieldName(), defaultColumn.getColumnName(), defaultColumn
- .getType(), defaultColumn.isID()));
+ sourceColumns.add(
+ new SourceColumn(defaultColumn.getFieldName(), defaultColumn.getColumnName(), defaultColumn
+ .getType(), defaultColumn.isID(), defaultColumn.getLength()));
}
return sourceColumns;
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpointrelation/EndpointRelationServerSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpointrelation/EndpointRelationServerSideMetrics.java
index c4d8c99..ea8fb58 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpointrelation/EndpointRelationServerSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpointrelation/EndpointRelationServerSideMetrics.java
@@ -52,7 +52,7 @@ public class EndpointRelationServerSideMetrics extends Metrics {
private String destEndpoint;
@Setter
@Getter
- @Column(columnName = COMPONENT_ID)
+ @Column(columnName = COMPONENT_ID, storageOnly = true)
@IDColumn
private int componentId;
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationClientSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationClientSideMetrics.java
index 6ec75a0..966cda9 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationClientSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationClientSideMetrics.java
@@ -64,7 +64,7 @@ public class ServiceInstanceRelationClientSideMetrics extends Metrics {
private int destServiceInstanceId;
@Setter
@Getter
- @Column(columnName = COMPONENT_ID)
+ @Column(columnName = COMPONENT_ID, storageOnly = true)
@IDColumn
private int componentId;
@Setter(AccessLevel.PRIVATE)
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java
index 38a5eec..3d08933 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java
@@ -64,7 +64,7 @@ public class ServiceInstanceRelationServerSideMetrics extends Metrics {
private int destServiceInstanceId;
@Setter
@Getter
- @Column(columnName = COMPONENT_ID)
+ @Column(columnName = COMPONENT_ID, storageOnly = true)
@IDColumn
private int componentId;
@Setter(AccessLevel.PRIVATE)
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationClientSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationClientSideMetrics.java
index cee1d36..daf8f1f 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationClientSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationClientSideMetrics.java
@@ -54,7 +54,7 @@ public class ServiceRelationClientSideMetrics extends Metrics {
private int destServiceId;
@Setter
@Getter
- @Column(columnName = COMPONENT_ID)
+ @Column(columnName = COMPONENT_ID, storageOnly = true)
@IDColumn
private int componentId;
@Setter(AccessLevel.PRIVATE)
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java
index 52c175b..094693e 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java
@@ -54,7 +54,7 @@ public class ServiceRelationServerSideMetrics extends Metrics {
private int destServiceId;
@Setter
@Getter
- @Column(columnName = COMPONENT_ID)
+ @Column(columnName = COMPONENT_ID, storageOnly = true)
@IDColumn
private int componentId;
@Setter(AccessLevel.PRIVATE)
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
index a36e69c..377abdf 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
@@ -52,11 +52,11 @@ public class SegmentRecord extends Record {
@Setter
@Getter
- @Column(columnName = SEGMENT_ID)
+ @Column(columnName = SEGMENT_ID, length = 150)
private String segmentId;
@Setter
@Getter
- @Column(columnName = TRACE_ID)
+ @Column(columnName = TRACE_ID, length = 150)
private String traceId;
@Setter
@Getter
@@ -92,11 +92,11 @@ public class SegmentRecord extends Record {
private int isError;
@Setter
@Getter
- @Column(columnName = DATA_BINARY)
+ @Column(columnName = DATA_BINARY, storageOnly = true)
private byte[] dataBinary;
@Setter
@Getter
- @Column(columnName = VERSION)
+ @Column(columnName = VERSION, storageOnly = true)
private int version;
@Override
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java
index 25dcae2..4ec46e8 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java
@@ -48,15 +48,15 @@ public abstract class ApdexMetrics extends Metrics implements IntValueHolder {
@Getter
@Setter
- @Column(columnName = TOTAL_NUM)
+ @Column(columnName = TOTAL_NUM, storageOnly = true)
private int totalNum;
@Getter
@Setter
- @Column(columnName = S_NUM)
+ @Column(columnName = S_NUM, storageOnly = true)
private int sNum;
@Getter
@Setter
- @Column(columnName = T_NUM)
+ @Column(columnName = T_NUM, storageOnly = true)
private int tNum;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/CPMMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/CPMMetrics.java
index 5c3f29f..e360b20 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/CPMMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/CPMMetrics.java
@@ -38,7 +38,7 @@ public abstract class CPMMetrics extends Metrics implements LongValueHolder {
private long value;
@Getter
@Setter
- @Column(columnName = TOTAL)
+ @Column(columnName = TOTAL, storageOnly = true)
private long total;
@Entrance
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java
index 07af20b..23bf84c 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java
@@ -36,11 +36,11 @@ public abstract class DoubleAvgMetrics extends Metrics implements DoubleValueHol
@Getter
@Setter
- @Column(columnName = SUMMATION)
+ @Column(columnName = SUMMATION, storageOnly = true)
private double summation;
@Getter
@Setter
- @Column(columnName = COUNT)
+ @Column(columnName = COUNT, storageOnly = true)
private long count;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/LongAvgMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/LongAvgMetrics.java
index 4773a75..e07383b 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/LongAvgMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/LongAvgMetrics.java
@@ -36,11 +36,11 @@ public abstract class LongAvgMetrics extends Metrics implements LongValueHolder
@Getter
@Setter
- @Column(columnName = SUMMATION)
+ @Column(columnName = SUMMATION, storageOnly = true)
private long summation;
@Getter
@Setter
- @Column(columnName = COUNT)
+ @Column(columnName = COUNT, storageOnly = true)
private long count;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java
index 2c0e2b2..8c8962b 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java
@@ -45,17 +45,21 @@ public abstract class PercentileMetrics extends GroupMetrics implements MultiInt
99
};
+ /**
+ * The special case when the column is isValue = true, but storageOnly = true, because it is {@link
+ * IntKeyLongValueHashMap} type, this column can't be query by the aggregation way.
+ */
@Getter
@Setter
- @Column(columnName = VALUE, isValue = true)
+ @Column(columnName = VALUE, isValue = true, storageOnly = true)
private IntKeyLongValueHashMap percentileValues;
@Getter
@Setter
- @Column(columnName = PRECISION)
+ @Column(columnName = PRECISION, storageOnly = true)
private int precision;
@Getter
@Setter
- @Column(columnName = DATASET)
+ @Column(columnName = DATASET, storageOnly = true)
private IntKeyLongValueHashMap dataset;
private boolean isCalculated;
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PxxMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PxxMetrics.java
index 658b757..9674eec 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PxxMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PxxMetrics.java
@@ -46,11 +46,11 @@ public abstract class PxxMetrics extends GroupMetrics implements IntValueHolder
private int value;
@Getter
@Setter
- @Column(columnName = PRECISION)
+ @Column(columnName = PRECISION, storageOnly = true)
private int precision;
@Getter
@Setter
- @Column(columnName = DETAIL_GROUP)
+ @Column(columnName = DETAIL_GROUP, storageOnly = true)
private IntKeyLongValueHashMap detailGroup;
private final int percentileRank;
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ThermodynamicMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ThermodynamicMetrics.java
index cc238db..967f594 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ThermodynamicMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ThermodynamicMetrics.java
@@ -43,15 +43,19 @@ public abstract class ThermodynamicMetrics extends GroupMetrics {
@Getter
@Setter
- @Column(columnName = STEP)
+ @Column(columnName = STEP, storageOnly = true)
private int step = 0;
@Getter
@Setter
- @Column(columnName = NUM_OF_STEPS)
+ @Column(columnName = NUM_OF_STEPS, storageOnly = true)
private int numOfSteps = 0;
+ /**
+ * The special case when the column is isValue = true, but storageOnly = true, because it is {@link
+ * IntKeyLongValueHashMap} type, this column can't be query by the aggregation way.
+ */
@Getter
@Setter
- @Column(columnName = DETAIL_GROUP, isValue = true)
+ @Column(columnName = DETAIL_GROUP, isValue = true, storageOnly = true)
private IntKeyLongValueHashMap detailGroup = new IntKeyLongValueHashMap(30);
/**
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/topn/TopN.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/topn/TopN.java
index bfc56e7..e7f9062 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/topn/TopN.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/topn/TopN.java
@@ -35,7 +35,7 @@ public abstract class TopN extends Record implements ComparableStorageData {
@Getter
@Setter
- @Column(columnName = STATEMENT, content = true)
+ @Column(columnName = STATEMENT, storageOnly = true)
private String statement;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java
index bfc8f5d..466bd89 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java
@@ -133,7 +133,7 @@ public class DefaultScopeDefine {
if (virtualColumn != null) {
scopeDefaultColumns.add(
new ScopeDefaultColumn(virtualColumn.fieldName(), virtualColumn.columnName(), virtualColumn
- .type(), virtualColumn.isID()));
+ .type(), virtualColumn.isID(), virtualColumn.length()));
}
Field[] scopeClassField = originalClass.getDeclaredFields();
if (scopeClassField != null) {
@@ -143,9 +143,9 @@ public class DefaultScopeDefine {
if (definedByField != null) {
if (!definedByField.requireDynamicActive() || ACTIVE_EXTRA_MODEL_COLUMNS) {
scopeDefaultColumns.add(
- new ScopeDefaultColumn(field.getName(), definedByField.columnName(), field.getType(),
- definedByField
- .isID()
+ new ScopeDefaultColumn(
+ field.getName(), definedByField.columnName(), field.getType(), false,
+ definedByField.length()
));
}
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java
index 45d5d74..67585ca 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java
@@ -24,25 +24,25 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import lombok.AccessLevel;
import lombok.Getter;
-import lombok.Setter;
/**
* Define the default columns of source scope. These columns pass down into the persistent entity(OAL metrics entity)
* automatically.
*/
@Getter(AccessLevel.PUBLIC)
-@Setter(AccessLevel.PUBLIC)
public class ScopeDefaultColumn {
private String fieldName;
private String columnName;
private Class<?> type;
private boolean isID;
+ private int length;
- public ScopeDefaultColumn(String fieldName, String columnName, Class<?> type, boolean isID) {
+ public ScopeDefaultColumn(String fieldName, String columnName, Class<?> type, boolean isID, int length) {
this.fieldName = fieldName;
this.columnName = columnName;
this.type = type;
this.isID = isID;
+ this.length = length;
}
@Target({ElementType.FIELD})
@@ -50,14 +50,17 @@ public class ScopeDefaultColumn {
public @interface DefinedByField {
String columnName();
- boolean isID() default false;
-
/**
* Dynamic active means this column is only activated through core setting explicitly.
*
* @return
*/
boolean requireDynamicActive() default false;
+
+ /**
+ * Define column length, only effective when the type is String.
+ */
+ int length() default 256;
}
@Target({ElementType.TYPE})
@@ -70,5 +73,10 @@ public class ScopeDefaultColumn {
Class type();
boolean isID() default false;
+
+ /**
+ * Define column length, only effective when the type is String.
+ */
+ int length() default 512;
}
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java
index d04e2c1..8c2f703 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java
@@ -55,5 +55,12 @@ public @interface Column {
/**
* The column is just saved, never used in query.
*/
- boolean content() default false;
+ boolean storageOnly() default false;
+
+ /**
+ * @return the length of this column, this is only for {@link String} column. The usage of this depends on the
+ * storage implementation.
+ * @since 7.1.0
+ */
+ int length() default 200;
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ColumnName.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ColumnName.java
index 6f84843..23944c8 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ColumnName.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ColumnName.java
@@ -18,10 +18,13 @@
package org.apache.skywalking.oap.server.core.storage.model;
+import lombok.ToString;
+
/**
* Short column name unsupported for now. No define in @Column annotation. The storage implementation need to use name
* to do match.
*/
+@ToString
public class ColumnName {
private String fullName;
private String storageName = null;
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java
index 16a6f66..387908e 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java
@@ -19,18 +19,42 @@
package org.apache.skywalking.oap.server.core.storage.model;
import lombok.Getter;
+import org.apache.skywalking.oap.server.core.analysis.metrics.IntKeyLongValueHashMap;
@Getter
public class ModelColumn {
private final ColumnName columnName;
private final Class<?> type;
private final boolean matchQuery;
- private final boolean content;
+ private final boolean storageOnly;
+ private final int length;
- public ModelColumn(ColumnName columnName, Class<?> type, boolean matchQuery, boolean content) {
+ public ModelColumn(ColumnName columnName,
+ Class<?> type,
+ boolean matchQuery,
+ boolean storageOnly,
+ boolean isValue,
+ int length) {
this.columnName = columnName;
this.type = type;
this.matchQuery = matchQuery;
- this.content = content;
+ this.length = length;
+ if (!type.equals(String.class) && length > 0) {
+ throw new IllegalArgumentException("Can't define the column " + columnName + " length.");
+ } else if (type.equals(String.class) && length == 0) {
+ throw new IllegalArgumentException("Can't define the column " + columnName + " length as 0.");
+ }
+ /**
+ * byte[] and {@link IntKeyLongValueHashMap} could never be query.
+ */
+ if (type.equals(byte[].class) || type.equals(IntKeyLongValueHashMap.class)) {
+ this.storageOnly = true;
+ } else {
+ if (storageOnly && isValue) {
+ throw new IllegalArgumentException(
+ "The column " + columnName + " can't be defined as both isValue and storageOnly.");
+ }
+ this.storageOnly = storageOnly;
+ }
}
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
index ad21f2a..80f364d 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
@@ -54,8 +54,11 @@ public class StorageModels implements IModelGetter, IModelSetter, IModelOverride
List<ModelColumn> modelColumns = new LinkedList<>();
retrieval(aClass, storage.getModelName(), modelColumns);
- Model model = new Model(storage.getModelName(), modelColumns, storage.isCapableOfTimeSeries(), storage.isDeleteHistory(), scopeId, storage
- .getDownsampling(), record);
+ Model model = new Model(
+ storage.getModelName(), modelColumns, storage.isCapableOfTimeSeries(), storage.isDeleteHistory(), scopeId,
+ storage
+ .getDownsampling(), record
+ );
models.add(model);
return model;
@@ -67,8 +70,9 @@ public class StorageModels implements IModelGetter, IModelSetter, IModelOverride
for (Field field : fields) {
if (field.isAnnotationPresent(Column.class)) {
Column column = field.getAnnotation(Column.class);
- modelColumns.add(new ModelColumn(new ColumnName(column.columnName()), field.getType(), column.matchQuery(), column
- .content()));
+ modelColumns.add(
+ new ModelColumn(new ColumnName(column.columnName()), field.getType(), column.matchQuery(), column
+ .storageOnly(), column.isValue(), column.length()));
if (logger.isDebugEnabled()) {
logger.debug("The field named {} with the {} type", column.columnName(), field.getType());
}
@@ -90,7 +94,8 @@ public class StorageModels implements IModelGetter, IModelSetter, IModelOverride
String name = existColumnName.getName();
if (name.equals(columnName)) {
existColumnName.setStorageName(newName);
- logger.debug("Model {} column {} has been override. The new column name is {}.", model.getName(), name, newName);
+ logger.debug(
+ "Model {} column {} has been override. The new column name is {}.", model.getName(), name, newName);
}
}));
}
diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java
new file mode 100644
index 0000000..659efa4
--- /dev/null
+++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.storage.model;
+
+import org.apache.skywalking.oap.server.core.analysis.metrics.IntKeyLongValueHashMap;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ModelColumnTest {
+ @Test
+ public void testColumnDefine() {
+ ModelColumn column = new ModelColumn(new ColumnName("abc"), byte[].class, true, false, true, 0);
+ Assert.assertEquals(true, column.isStorageOnly());
+ Assert.assertEquals("abc", column.getColumnName().getName());
+
+ column = new ModelColumn(new ColumnName("abc"), IntKeyLongValueHashMap.class, true, false, true, 0);
+ Assert.assertEquals(true, column.isStorageOnly());
+ Assert.assertEquals("abc", column.getColumnName().getName());
+
+ column = new ModelColumn(new ColumnName("abc"), String.class, true, false, true, 200);
+ Assert.assertEquals(false, column.isStorageOnly());
+ Assert.assertEquals("abc", column.getColumnName().getName());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testConflictDefinition() {
+ ModelColumn column = new ModelColumn(new ColumnName("abc"), String.class, true, true, true, 200);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testStringColumnNoLength() {
+ ModelColumn column = new ModelColumn(new ColumnName("abc"), String.class, true, true, true, 0);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIntColumnWithLength() {
+ ModelColumn column = new ModelColumn(new ColumnName("abc"), int.class, true, true, true, 100);
+ }
+}
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
index 20c3b65..422b857 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
@@ -139,14 +139,12 @@ public class StorageEsInstaller extends ModelInstaller {
matchColumn.put("type", "text");
matchColumn.put("analyzer", "oap_analyzer");
properties.put(matchCName, matchColumn);
- } else if (columnDefine.isContent()) {
- Map<String, Object> column = new HashMap<>();
- column.put("type", "text");
- column.put("index", false);
- properties.put(columnDefine.getColumnName().getName(), column);
} else {
Map<String, Object> column = new HashMap<>();
column.put("type", columnTypeEsMapping.transform(columnDefine.getType()));
+ if (columnDefine.isStorageOnly()) {
+ column.put("index", false);
+ }
properties.put(columnDefine.getColumnName().getName(), column);
}
}
diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java
index 38dfd00..ef76958 100644
--- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java
+++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java
@@ -149,8 +149,10 @@ public class InfluxStorageProvider extends ModuleProvider {
ModelInstaller installer;
if (config.getMetabaseType().equalsIgnoreCase("h2")) {
installer = new InfluxDBH2MetaDBInstaller(getManager());
- } else {
+ } else if (config.getMetabaseType().equalsIgnoreCase("mysql")) {
installer = new InfluxDBMySQLMetaDBInstaller(getManager());
+ } else {
+ throw new IllegalArgumentException("Unavailable metabase type, " + config.getMetabaseType());
}
installer.install(client);
new H2RegisterLockInstaller().install(client, lockDAO);
diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TableInstaller.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TableInstaller.java
index f16f8b3..150b717 100644
--- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TableInstaller.java
+++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TableInstaller.java
@@ -74,7 +74,7 @@ public class H2TableInstaller extends ModelInstaller {
ModelColumn column = model.getColumns().get(i);
ColumnName name = column.getColumnName();
tableCreateSQL.appendLine(
- name.getStorageName() + " " + getColumnType(model, name, column.getType()) + (i != model
+ name.getStorageName() + " " + getColumnType(model, column) + (i != model
.getColumns()
.size() - 1 ? "," : ""));
}
@@ -97,7 +97,8 @@ public class H2TableInstaller extends ModelInstaller {
/**
* Set up the data type mapping between Java type and H2 database type
*/
- protected String getColumnType(Model model, ColumnName name, Class<?> type) {
+ protected String getColumnType(Model model, ModelColumn column) {
+ final Class<?> type = column.getType();
if (Integer.class.equals(type) || int.class.equals(type)) {
return "INT";
} else if (Long.class.equals(type) || long.class.equals(type)) {
@@ -105,16 +106,11 @@ public class H2TableInstaller extends ModelInstaller {
} else if (Double.class.equals(type) || double.class.equals(type)) {
return "DOUBLE";
} else if (String.class.equals(type)) {
- return "VARCHAR(2000)";
+ return "VARCHAR(" + column.getLength() + ")";
} else if (IntKeyLongValueHashMap.class.equals(type)) {
- return "VARCHAR(20000)";
+ return "MEDIUMTEXT";
} else if (byte[].class.equals(type)) {
- if (DefaultScopeDefine.SEGMENT == model.getScopeId()) {
- if (name.getName().equals(SegmentRecord.DATA_BINARY)) {
- return "MEDIUMTEXT";
- }
- }
- return "VARCHAR(20000)";
+ return "MEDIUMTEXT";
} else {
throw new IllegalArgumentException("Unsupported data type: " + type.getName());
}
diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java
index f0320c2..f69eb07 100644
--- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java
+++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java
@@ -23,13 +23,11 @@ import java.sql.SQLException;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.core.alarm.AlarmRecord;
import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord;
-import org.apache.skywalking.oap.server.core.analysis.metrics.IntKeyLongValueHashMap;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.profile.ProfileTaskLogRecord;
import org.apache.skywalking.oap.server.core.profile.ProfileThreadSnapshotRecord;
import org.apache.skywalking.oap.server.core.register.RegisterSource;
import org.apache.skywalking.oap.server.core.storage.StorageException;
-import org.apache.skywalking.oap.server.core.storage.model.ColumnName;
import org.apache.skywalking.oap.server.core.storage.model.Model;
import org.apache.skywalking.oap.server.library.client.Client;
import org.apache.skywalking.oap.server.library.client.jdbc.JDBCClientException;
@@ -68,43 +66,6 @@ public class MySQLTableInstaller extends H2TableInstaller {
}
/**
- * Based on MySQL features, provide a specific data type mappings.
- */
- @Override
- protected String getColumnType(Model model, ColumnName name, Class<?> type) {
- if (Integer.class.equals(type) || int.class.equals(type)) {
- return "INT";
- } else if (Long.class.equals(type) || long.class.equals(type)) {
- return "BIGINT";
- } else if (Double.class.equals(type) || double.class.equals(type)) {
- return "DOUBLE";
- } else if (String.class.equals(type)) {
- if (name.getName().equals(SegmentRecord.TRACE_ID) || name.getName().equals(SegmentRecord.SEGMENT_ID)) {
- return "VARCHAR(150)";
- }
- if (Metrics.ENTITY_ID.equals(name.getName())) {
- return "VARCHAR(512)";
- }
- if (SegmentRecord.ENDPOINT_NAME.equals(name.getName()) || SegmentRecord.ENDPOINT_ID.equals(
- name.getName())) {
- return "VARCHAR(200)";
- }
- if (PROFILE_TASK_LOG == model.getScopeId() || PROFILE_TASK_SEGMENT_SNAPSHOT == model.getScopeId()) {
- if (name.getName().equals(ProfileTaskLogRecord.TASK_ID)) {
- return "VARCHAR(300)";
- }
- }
- return "VARCHAR(2000)";
- } else if (IntKeyLongValueHashMap.class.equals(type)) {
- return "MEDIUMTEXT";
- } else if (byte[].class.equals(type)) {
- return "MEDIUMTEXT";
- } else {
- throw new IllegalArgumentException("Unsupported data type: " + type.getName());
- }
- }
-
- /**
* Create indexes of all tables. Due to MySQL storage is suitable for middle size use case and also compatible with
* TiDB users, Indexes are required for the UI query.
*