You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2020/03/29 01:58:46 UTC
[skywalking] branch core-refactor created (now 4798983)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a change to branch core-refactor
in repository https://gitbox.apache.org/repos/asf/skywalking.git.
at 4798983 1. Add length definition with reasonable default value. 2. Merge H2 and MySQL columntype mapping back to consistent.
This branch includes the following new commits:
new 4798983 1. Add length definition with reasonable default value. 2. Merge H2 and MySQL columntype mapping back to consistent.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[skywalking] 01/01: 1. Add length definition with reasonable
default value. 2. Merge H2 and MySQL columntype mapping back to consistent.
Posted by wu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch core-refactor
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 4798983c4f73e6d52b5a6f85a353c8b1776363c9
Author: Wu Sheng <wu...@foxmail.com>
AuthorDate: Sun Mar 29 09:58:07 2020 +0800
1. Add length definition with reasonable default value.
2. Merge H2 and MySQL columntype mapping back to consistent.
---
.../org/apache/skywalking/oal/rt/OALRuntime.java | 47 +++++++++++++-----
.../skywalking/oal/rt/parser/SourceColumn.java | 4 +-
.../oal/rt/parser/SourceColumnsFactory.java | 5 +-
.../EndpointRelationServerSideMetrics.java | 2 +-
.../ServiceInstanceRelationClientSideMetrics.java | 2 +-
.../ServiceInstanceRelationServerSideMetrics.java | 2 +-
.../service/ServiceRelationClientSideMetrics.java | 2 +-
.../service/ServiceRelationServerSideMetrics.java | 2 +-
.../analysis/manual/segment/SegmentRecord.java | 8 ++--
.../server/core/analysis/metrics/ApdexMetrics.java | 6 +--
.../server/core/analysis/metrics/CPMMetrics.java | 2 +-
.../core/analysis/metrics/DoubleAvgMetrics.java | 4 +-
.../core/analysis/metrics/LongAvgMetrics.java | 4 +-
.../core/analysis/metrics/PercentileMetrics.java | 10 ++--
.../server/core/analysis/metrics/PxxMetrics.java | 4 +-
.../analysis/metrics/ThermodynamicMetrics.java | 10 ++--
.../oap/server/core/analysis/topn/TopN.java | 2 +-
.../oap/server/core/source/DefaultScopeDefine.java | 8 ++--
.../oap/server/core/source/ScopeDefaultColumn.java | 18 +++++--
.../oap/server/core/storage/annotation/Column.java | 9 +++-
.../oap/server/core/storage/model/ColumnName.java | 3 ++
.../oap/server/core/storage/model/ModelColumn.java | 30 ++++++++++--
.../server/core/storage/model/StorageModels.java | 15 ++++--
.../server/core/storage/model/ModelColumnTest.java | 55 ++++++++++++++++++++++
.../elasticsearch/base/StorageEsInstaller.java | 8 ++--
.../plugin/influxdb/InfluxStorageProvider.java | 4 +-
.../plugin/jdbc/h2/dao/H2TableInstaller.java | 16 +++----
.../plugin/jdbc/mysql/MySQLTableInstaller.java | 39 ---------------
28 files changed, 206 insertions(+), 115 deletions(-)
diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java
index 33d6fa3..6a7cde1 100644
--- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java
+++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/OALRuntime.java
@@ -236,20 +236,25 @@ public class OALRuntime implements OALEngine {
*/
for (SourceColumn field : metricsStmt.getFieldsFromSource()) {
try {
- CtField newField = CtField.make("private " + field.getType()
- .getName() + " " + field.getFieldName() + ";", metricsClass);
+ CtField newField = CtField.make(
+ "private " + field.getType()
+ .getName() + " " + field.getFieldName() + ";", metricsClass);
metricsClass.addField(newField);
metricsClass.addMethod(CtNewMethod.getter(field.getFieldGetter(), newField));
metricsClass.addMethod(CtNewMethod.setter(field.getFieldSetter(), newField));
- AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(constPool, AnnotationsAttribute.visibleTag);
+ AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(
+ constPool, AnnotationsAttribute.visibleTag);
/**
* Add @Column(columnName = "${sourceField.columnName}")
*/
Annotation columnAnnotation = new Annotation(Column.class.getName(), constPool);
columnAnnotation.addMemberValue("columnName", new StringMemberValue(field.getColumnName(), constPool));
+ if (field.getType().equals(String.class)) {
+ columnAnnotation.addMemberValue("length", new IntegerMemberValue(field.getLength(), constPool));
+ }
annotationsAttribute.addAnnotation(columnAnnotation);
if (field.isID()) {
@@ -263,7 +268,8 @@ public class OALRuntime implements OALEngine {
newField.getFieldInfo().addAttribute(annotationsAttribute);
} catch (CannotCompileException e) {
- logger.error("Can't add field(including set/get) " + field.getFieldName() + " in " + className + ".", e);
+ logger.error(
+ "Can't add field(including set/get) " + field.getFieldName() + " in " + className + ".", e);
throw new OALCompileException(e.getMessage(), e);
}
}
@@ -287,11 +293,13 @@ public class OALRuntime implements OALEngine {
*
* at Stream(name = "${tableName}", scopeId = ${sourceScopeId}, builder = ${metricsName}Metrics.Builder.class, processor = MetricsStreamProcessor.class)
*/
- AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(constPool, AnnotationsAttribute.visibleTag);
+ AnnotationsAttribute annotationsAttribute = new AnnotationsAttribute(
+ constPool, AnnotationsAttribute.visibleTag);
Annotation streamAnnotation = new Annotation(Stream.class.getName(), constPool);
streamAnnotation.addMemberValue("name", new StringMemberValue(metricsStmt.getTableName(), constPool));
streamAnnotation.addMemberValue("scopeId", new IntegerMemberValue(constPool, metricsStmt.getSourceScopeId()));
- streamAnnotation.addMemberValue("builder", new ClassMemberValue(metricsBuilderClassName(metricsStmt, true), constPool));
+ streamAnnotation.addMemberValue(
+ "builder", new ClassMemberValue(metricsBuilderClassName(metricsStmt, true), constPool));
streamAnnotation.addMemberValue("processor", new ClassMemberValue(METRICS_STREAM_PROCESSOR, constPool));
annotationsAttribute.addAnnotation(streamAnnotation);
@@ -328,7 +336,8 @@ public class OALRuntime implements OALEngine {
* Create empty construct
*/
try {
- CtConstructor defaultConstructor = CtNewConstructor.make("public " + className + "() {}", metricsBuilderClass);
+ CtConstructor defaultConstructor = CtNewConstructor.make(
+ "public " + className + "() {}", metricsBuilderClass);
metricsBuilderClass.addConstructor(defaultConstructor);
} catch (CannotCompileException e) {
logger.error("Can't add empty constructor in " + className + ".", e);
@@ -363,7 +372,7 @@ public class OALRuntime implements OALEngine {
* Generate SourceDispatcher class and inject it to classloader
*/
private Class generateDispatcherClass(String scopeName,
- DispatcherContext dispatcherContext) throws OALCompileException {
+ DispatcherContext dispatcherContext) throws OALCompileException {
String className = dispatcherClassName(scopeName, false);
CtClass dispatcherClass = classPool.makeClass(dispatcherClassName(scopeName, true));
@@ -377,10 +386,19 @@ public class OALRuntime implements OALEngine {
*/
String sourceClassName = SOURCE_PACKAGE + dispatcherContext.getSource();
SignatureAttribute.ClassSignature dispatcherSignature = new SignatureAttribute.ClassSignature(null, null,
- // Set interface and its generic params
- new SignatureAttribute.ClassType[] {
- new SignatureAttribute.ClassType(SourceDispatcher.class.getCanonicalName(), new SignatureAttribute.TypeArgument[] {new SignatureAttribute.TypeArgument(new SignatureAttribute.ClassType(sourceClassName))})
- });
+ // Set interface and its generic params
+ new SignatureAttribute.ClassType[] {
+ new SignatureAttribute.ClassType(
+ SourceDispatcher.class
+ .getCanonicalName(),
+ new SignatureAttribute.TypeArgument[] {
+ new SignatureAttribute.TypeArgument(
+ new SignatureAttribute.ClassType(
+ sourceClassName))
+ }
+ )
+ }
+ );
dispatcherClass.setGenericSignature(dispatcherSignature.encode());
} catch (NotFoundException e) {
@@ -397,7 +415,10 @@ public class OALRuntime implements OALEngine {
configuration.getTemplate("dispatcher/doMetrics.ftl").process(dispatcherContextMetric, methodEntity);
dispatcherClass.addMethod(CtNewMethod.make(methodEntity.toString(), dispatcherClass));
} catch (Exception e) {
- logger.error("Can't generate method do" + dispatcherContextMetric.getMetricsName() + " for " + className + ".", e);
+ logger.error(
+ "Can't generate method do" + dispatcherContextMetric.getMetricsName() + " for " + className + ".",
+ e
+ );
logger.error("Method body as following" + System.lineSeparator() + "{}", methodEntity);
throw new OALCompileException(e.getMessage(), e);
}
diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumn.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumn.java
index 59d442a..fc798a3 100644
--- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumn.java
+++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumn.java
@@ -32,15 +32,17 @@ public class SourceColumn {
private Class<?> type;
private String typeName;
private boolean isID;
+ private int length;
private String fieldSetter;
private String fieldGetter;
- public SourceColumn(String fieldName, String columnName, Class<?> type, boolean isID) {
+ public SourceColumn(String fieldName, String columnName, Class<?> type, boolean isID, int length) {
this.fieldName = fieldName;
this.columnName = columnName;
this.type = type;
this.typeName = type.getName();
this.isID = isID;
+ this.length = length;
this.fieldGetter = ClassMethodUtil.toGetMethod(fieldName);
this.fieldSetter = ClassMethodUtil.toSetMethod(fieldName);
diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumnsFactory.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumnsFactory.java
index 12f1cac..2339513 100644
--- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumnsFactory.java
+++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/SourceColumnsFactory.java
@@ -29,8 +29,9 @@ public class SourceColumnsFactory {
List<ScopeDefaultColumn> columns = DefaultScopeDefine.getDefaultColumns(source);
for (ScopeDefaultColumn defaultColumn : columns) {
- sourceColumns.add(new SourceColumn(defaultColumn.getFieldName(), defaultColumn.getColumnName(), defaultColumn
- .getType(), defaultColumn.isID()));
+ sourceColumns.add(
+ new SourceColumn(defaultColumn.getFieldName(), defaultColumn.getColumnName(), defaultColumn
+ .getType(), defaultColumn.isID(), defaultColumn.getLength()));
}
return sourceColumns;
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpointrelation/EndpointRelationServerSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpointrelation/EndpointRelationServerSideMetrics.java
index c4d8c99..ea8fb58 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpointrelation/EndpointRelationServerSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/endpointrelation/EndpointRelationServerSideMetrics.java
@@ -52,7 +52,7 @@ public class EndpointRelationServerSideMetrics extends Metrics {
private String destEndpoint;
@Setter
@Getter
- @Column(columnName = COMPONENT_ID)
+ @Column(columnName = COMPONENT_ID, storageOnly = true)
@IDColumn
private int componentId;
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationClientSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationClientSideMetrics.java
index 6ec75a0..966cda9 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationClientSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationClientSideMetrics.java
@@ -64,7 +64,7 @@ public class ServiceInstanceRelationClientSideMetrics extends Metrics {
private int destServiceInstanceId;
@Setter
@Getter
- @Column(columnName = COMPONENT_ID)
+ @Column(columnName = COMPONENT_ID, storageOnly = true)
@IDColumn
private int componentId;
@Setter(AccessLevel.PRIVATE)
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java
index 38a5eec..3d08933 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java
@@ -64,7 +64,7 @@ public class ServiceInstanceRelationServerSideMetrics extends Metrics {
private int destServiceInstanceId;
@Setter
@Getter
- @Column(columnName = COMPONENT_ID)
+ @Column(columnName = COMPONENT_ID, storageOnly = true)
@IDColumn
private int componentId;
@Setter(AccessLevel.PRIVATE)
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationClientSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationClientSideMetrics.java
index cee1d36..daf8f1f 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationClientSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationClientSideMetrics.java
@@ -54,7 +54,7 @@ public class ServiceRelationClientSideMetrics extends Metrics {
private int destServiceId;
@Setter
@Getter
- @Column(columnName = COMPONENT_ID)
+ @Column(columnName = COMPONENT_ID, storageOnly = true)
@IDColumn
private int componentId;
@Setter(AccessLevel.PRIVATE)
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java
index 52c175b..094693e 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java
@@ -54,7 +54,7 @@ public class ServiceRelationServerSideMetrics extends Metrics {
private int destServiceId;
@Setter
@Getter
- @Column(columnName = COMPONENT_ID)
+ @Column(columnName = COMPONENT_ID, storageOnly = true)
@IDColumn
private int componentId;
@Setter(AccessLevel.PRIVATE)
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
index a36e69c..377abdf 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/segment/SegmentRecord.java
@@ -52,11 +52,11 @@ public class SegmentRecord extends Record {
@Setter
@Getter
- @Column(columnName = SEGMENT_ID)
+ @Column(columnName = SEGMENT_ID, length = 150)
private String segmentId;
@Setter
@Getter
- @Column(columnName = TRACE_ID)
+ @Column(columnName = TRACE_ID, length = 150)
private String traceId;
@Setter
@Getter
@@ -92,11 +92,11 @@ public class SegmentRecord extends Record {
private int isError;
@Setter
@Getter
- @Column(columnName = DATA_BINARY)
+ @Column(columnName = DATA_BINARY, storageOnly = true)
private byte[] dataBinary;
@Setter
@Getter
- @Column(columnName = VERSION)
+ @Column(columnName = VERSION, storageOnly = true)
private int version;
@Override
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java
index 25dcae2..4ec46e8 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java
@@ -48,15 +48,15 @@ public abstract class ApdexMetrics extends Metrics implements IntValueHolder {
@Getter
@Setter
- @Column(columnName = TOTAL_NUM)
+ @Column(columnName = TOTAL_NUM, storageOnly = true)
private int totalNum;
@Getter
@Setter
- @Column(columnName = S_NUM)
+ @Column(columnName = S_NUM, storageOnly = true)
private int sNum;
@Getter
@Setter
- @Column(columnName = T_NUM)
+ @Column(columnName = T_NUM, storageOnly = true)
private int tNum;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/CPMMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/CPMMetrics.java
index 5c3f29f..e360b20 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/CPMMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/CPMMetrics.java
@@ -38,7 +38,7 @@ public abstract class CPMMetrics extends Metrics implements LongValueHolder {
private long value;
@Getter
@Setter
- @Column(columnName = TOTAL)
+ @Column(columnName = TOTAL, storageOnly = true)
private long total;
@Entrance
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java
index 07af20b..23bf84c 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java
@@ -36,11 +36,11 @@ public abstract class DoubleAvgMetrics extends Metrics implements DoubleValueHol
@Getter
@Setter
- @Column(columnName = SUMMATION)
+ @Column(columnName = SUMMATION, storageOnly = true)
private double summation;
@Getter
@Setter
- @Column(columnName = COUNT)
+ @Column(columnName = COUNT, storageOnly = true)
private long count;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/LongAvgMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/LongAvgMetrics.java
index 4773a75..e07383b 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/LongAvgMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/LongAvgMetrics.java
@@ -36,11 +36,11 @@ public abstract class LongAvgMetrics extends Metrics implements LongValueHolder
@Getter
@Setter
- @Column(columnName = SUMMATION)
+ @Column(columnName = SUMMATION, storageOnly = true)
private long summation;
@Getter
@Setter
- @Column(columnName = COUNT)
+ @Column(columnName = COUNT, storageOnly = true)
private long count;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java
index 2c0e2b2..8c8962b 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java
@@ -45,17 +45,21 @@ public abstract class PercentileMetrics extends GroupMetrics implements MultiInt
99
};
+ /**
+ * The special case when the column is isValue = true, but storageOnly = true, because it is {@link
+ * IntKeyLongValueHashMap} type, this column can't be query by the aggregation way.
+ */
@Getter
@Setter
- @Column(columnName = VALUE, isValue = true)
+ @Column(columnName = VALUE, isValue = true, storageOnly = true)
private IntKeyLongValueHashMap percentileValues;
@Getter
@Setter
- @Column(columnName = PRECISION)
+ @Column(columnName = PRECISION, storageOnly = true)
private int precision;
@Getter
@Setter
- @Column(columnName = DATASET)
+ @Column(columnName = DATASET, storageOnly = true)
private IntKeyLongValueHashMap dataset;
private boolean isCalculated;
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PxxMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PxxMetrics.java
index 658b757..9674eec 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PxxMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PxxMetrics.java
@@ -46,11 +46,11 @@ public abstract class PxxMetrics extends GroupMetrics implements IntValueHolder
private int value;
@Getter
@Setter
- @Column(columnName = PRECISION)
+ @Column(columnName = PRECISION, storageOnly = true)
private int precision;
@Getter
@Setter
- @Column(columnName = DETAIL_GROUP)
+ @Column(columnName = DETAIL_GROUP, storageOnly = true)
private IntKeyLongValueHashMap detailGroup;
private final int percentileRank;
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ThermodynamicMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ThermodynamicMetrics.java
index cc238db..967f594 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ThermodynamicMetrics.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ThermodynamicMetrics.java
@@ -43,15 +43,19 @@ public abstract class ThermodynamicMetrics extends GroupMetrics {
@Getter
@Setter
- @Column(columnName = STEP)
+ @Column(columnName = STEP, storageOnly = true)
private int step = 0;
@Getter
@Setter
- @Column(columnName = NUM_OF_STEPS)
+ @Column(columnName = NUM_OF_STEPS, storageOnly = true)
private int numOfSteps = 0;
+ /**
+ * The special case when the column is isValue = true, but storageOnly = true, because it is {@link
+ * IntKeyLongValueHashMap} type, this column can't be query by the aggregation way.
+ */
@Getter
@Setter
- @Column(columnName = DETAIL_GROUP, isValue = true)
+ @Column(columnName = DETAIL_GROUP, isValue = true, storageOnly = true)
private IntKeyLongValueHashMap detailGroup = new IntKeyLongValueHashMap(30);
/**
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/topn/TopN.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/topn/TopN.java
index bfc56e7..e7f9062 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/topn/TopN.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/topn/TopN.java
@@ -35,7 +35,7 @@ public abstract class TopN extends Record implements ComparableStorageData {
@Getter
@Setter
- @Column(columnName = STATEMENT, content = true)
+ @Column(columnName = STATEMENT, storageOnly = true)
private String statement;
@Getter
@Setter
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java
index bfc8f5d..466bd89 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java
@@ -133,7 +133,7 @@ public class DefaultScopeDefine {
if (virtualColumn != null) {
scopeDefaultColumns.add(
new ScopeDefaultColumn(virtualColumn.fieldName(), virtualColumn.columnName(), virtualColumn
- .type(), virtualColumn.isID()));
+ .type(), virtualColumn.isID(), virtualColumn.length()));
}
Field[] scopeClassField = originalClass.getDeclaredFields();
if (scopeClassField != null) {
@@ -143,9 +143,9 @@ public class DefaultScopeDefine {
if (definedByField != null) {
if (!definedByField.requireDynamicActive() || ACTIVE_EXTRA_MODEL_COLUMNS) {
scopeDefaultColumns.add(
- new ScopeDefaultColumn(field.getName(), definedByField.columnName(), field.getType(),
- definedByField
- .isID()
+ new ScopeDefaultColumn(
+ field.getName(), definedByField.columnName(), field.getType(), false,
+ definedByField.length()
));
}
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java
index 45d5d74..67585ca 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java
@@ -24,25 +24,25 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import lombok.AccessLevel;
import lombok.Getter;
-import lombok.Setter;
/**
* Define the default columns of source scope. These columns pass down into the persistent entity(OAL metrics entity)
* automatically.
*/
@Getter(AccessLevel.PUBLIC)
-@Setter(AccessLevel.PUBLIC)
public class ScopeDefaultColumn {
private String fieldName;
private String columnName;
private Class<?> type;
private boolean isID;
+ private int length;
- public ScopeDefaultColumn(String fieldName, String columnName, Class<?> type, boolean isID) {
+ public ScopeDefaultColumn(String fieldName, String columnName, Class<?> type, boolean isID, int length) {
this.fieldName = fieldName;
this.columnName = columnName;
this.type = type;
this.isID = isID;
+ this.length = length;
}
@Target({ElementType.FIELD})
@@ -50,14 +50,17 @@ public class ScopeDefaultColumn {
public @interface DefinedByField {
String columnName();
- boolean isID() default false;
-
/**
* Dynamic active means this column is only activated through core setting explicitly.
*
* @return
*/
boolean requireDynamicActive() default false;
+
+ /**
+ * Define column length, only effective when the type is String.
+ */
+ int length() default 256;
}
@Target({ElementType.TYPE})
@@ -70,5 +73,10 @@ public class ScopeDefaultColumn {
Class type();
boolean isID() default false;
+
+ /**
+ * Define column length, only effective when the type is String.
+ */
+ int length() default 512;
}
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java
index d04e2c1..8c2f703 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java
@@ -55,5 +55,12 @@ public @interface Column {
/**
* The column is just saved, never used in query.
*/
- boolean content() default false;
+ boolean storageOnly() default false;
+
+ /**
+ * @return the length of this column, this is only for {@link String} column. The usage of this depends on the
+ * storage implementation.
+ * @since 7.1.0
+ */
+ int length() default 200;
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ColumnName.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ColumnName.java
index 6f84843..23944c8 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ColumnName.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ColumnName.java
@@ -18,10 +18,13 @@
package org.apache.skywalking.oap.server.core.storage.model;
+import lombok.ToString;
+
/**
* Short column name unsupported for now. No define in @Column annotation. The storage implementation need to use name
* to do match.
*/
+@ToString
public class ColumnName {
private String fullName;
private String storageName = null;
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java
index 16a6f66..387908e 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumn.java
@@ -19,18 +19,42 @@
package org.apache.skywalking.oap.server.core.storage.model;
import lombok.Getter;
+import org.apache.skywalking.oap.server.core.analysis.metrics.IntKeyLongValueHashMap;
@Getter
public class ModelColumn {
private final ColumnName columnName;
private final Class<?> type;
private final boolean matchQuery;
- private final boolean content;
+ private final boolean storageOnly;
+ private final int length;
- public ModelColumn(ColumnName columnName, Class<?> type, boolean matchQuery, boolean content) {
+ public ModelColumn(ColumnName columnName,
+ Class<?> type,
+ boolean matchQuery,
+ boolean storageOnly,
+ boolean isValue,
+ int length) {
this.columnName = columnName;
this.type = type;
this.matchQuery = matchQuery;
- this.content = content;
+ this.length = length;
+ if (!type.equals(String.class) && length > 0) {
+ throw new IllegalArgumentException("Can't define the column " + columnName + " length.");
+ } else if (type.equals(String.class) && length == 0) {
+ throw new IllegalArgumentException("Can't define the column " + columnName + " length as 0.");
+ }
+ /**
+ * byte[] and {@link IntKeyLongValueHashMap} could never be query.
+ */
+ if (type.equals(byte[].class) || type.equals(IntKeyLongValueHashMap.class)) {
+ this.storageOnly = true;
+ } else {
+ if (storageOnly && isValue) {
+ throw new IllegalArgumentException(
+ "The column " + columnName + " can't be defined as both isValue and storageOnly.");
+ }
+ this.storageOnly = storageOnly;
+ }
}
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
index ad21f2a..80f364d 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java
@@ -54,8 +54,11 @@ public class StorageModels implements IModelGetter, IModelSetter, IModelOverride
List<ModelColumn> modelColumns = new LinkedList<>();
retrieval(aClass, storage.getModelName(), modelColumns);
- Model model = new Model(storage.getModelName(), modelColumns, storage.isCapableOfTimeSeries(), storage.isDeleteHistory(), scopeId, storage
- .getDownsampling(), record);
+ Model model = new Model(
+ storage.getModelName(), modelColumns, storage.isCapableOfTimeSeries(), storage.isDeleteHistory(), scopeId,
+ storage
+ .getDownsampling(), record
+ );
models.add(model);
return model;
@@ -67,8 +70,9 @@ public class StorageModels implements IModelGetter, IModelSetter, IModelOverride
for (Field field : fields) {
if (field.isAnnotationPresent(Column.class)) {
Column column = field.getAnnotation(Column.class);
- modelColumns.add(new ModelColumn(new ColumnName(column.columnName()), field.getType(), column.matchQuery(), column
- .content()));
+ modelColumns.add(
+ new ModelColumn(new ColumnName(column.columnName()), field.getType(), column.matchQuery(), column
+ .storageOnly(), column.isValue(), column.length()));
if (logger.isDebugEnabled()) {
logger.debug("The field named {} with the {} type", column.columnName(), field.getType());
}
@@ -90,7 +94,8 @@ public class StorageModels implements IModelGetter, IModelSetter, IModelOverride
String name = existColumnName.getName();
if (name.equals(columnName)) {
existColumnName.setStorageName(newName);
- logger.debug("Model {} column {} has been override. The new column name is {}.", model.getName(), name, newName);
+ logger.debug(
+ "Model {} column {} has been override. The new column name is {}.", model.getName(), name, newName);
}
}));
}
diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java
new file mode 100644
index 0000000..659efa4
--- /dev/null
+++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/model/ModelColumnTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.storage.model;
+
+import org.apache.skywalking.oap.server.core.analysis.metrics.IntKeyLongValueHashMap;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ModelColumnTest {
+ @Test
+ public void testColumnDefine() {
+ ModelColumn column = new ModelColumn(new ColumnName("abc"), byte[].class, true, false, true, 0);
+ Assert.assertEquals(true, column.isStorageOnly());
+ Assert.assertEquals("abc", column.getColumnName().getName());
+
+ column = new ModelColumn(new ColumnName("abc"), IntKeyLongValueHashMap.class, true, false, true, 0);
+ Assert.assertEquals(true, column.isStorageOnly());
+ Assert.assertEquals("abc", column.getColumnName().getName());
+
+ column = new ModelColumn(new ColumnName("abc"), String.class, true, false, true, 200);
+ Assert.assertEquals(false, column.isStorageOnly());
+ Assert.assertEquals("abc", column.getColumnName().getName());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testConflictDefinition() {
+ ModelColumn column = new ModelColumn(new ColumnName("abc"), String.class, true, true, true, 200);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testStringColumnNoLength() {
+ ModelColumn column = new ModelColumn(new ColumnName("abc"), String.class, true, true, true, 0);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIntColumnWithLength() {
+ ModelColumn column = new ModelColumn(new ColumnName("abc"), int.class, true, true, true, 100);
+ }
+}
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
index 20c3b65..422b857 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
@@ -139,14 +139,12 @@ public class StorageEsInstaller extends ModelInstaller {
matchColumn.put("type", "text");
matchColumn.put("analyzer", "oap_analyzer");
properties.put(matchCName, matchColumn);
- } else if (columnDefine.isContent()) {
- Map<String, Object> column = new HashMap<>();
- column.put("type", "text");
- column.put("index", false);
- properties.put(columnDefine.getColumnName().getName(), column);
} else {
Map<String, Object> column = new HashMap<>();
column.put("type", columnTypeEsMapping.transform(columnDefine.getType()));
+ if (columnDefine.isStorageOnly()) {
+ column.put("index", false);
+ }
properties.put(columnDefine.getColumnName().getName(), column);
}
}
diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java
index 38dfd00..ef76958 100644
--- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java
+++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java
@@ -149,8 +149,10 @@ public class InfluxStorageProvider extends ModuleProvider {
ModelInstaller installer;
if (config.getMetabaseType().equalsIgnoreCase("h2")) {
installer = new InfluxDBH2MetaDBInstaller(getManager());
- } else {
+ } else if (config.getMetabaseType().equalsIgnoreCase("mysql")) {
installer = new InfluxDBMySQLMetaDBInstaller(getManager());
+ } else {
+ throw new IllegalArgumentException("Unavailable metabase type, " + config.getMetabaseType());
}
installer.install(client);
new H2RegisterLockInstaller().install(client, lockDAO);
diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TableInstaller.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TableInstaller.java
index f16f8b3..150b717 100644
--- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TableInstaller.java
+++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TableInstaller.java
@@ -74,7 +74,7 @@ public class H2TableInstaller extends ModelInstaller {
ModelColumn column = model.getColumns().get(i);
ColumnName name = column.getColumnName();
tableCreateSQL.appendLine(
- name.getStorageName() + " " + getColumnType(model, name, column.getType()) + (i != model
+ name.getStorageName() + " " + getColumnType(model, column) + (i != model
.getColumns()
.size() - 1 ? "," : ""));
}
@@ -97,7 +97,8 @@ public class H2TableInstaller extends ModelInstaller {
/**
* Set up the data type mapping between Java type and H2 database type
*/
- protected String getColumnType(Model model, ColumnName name, Class<?> type) {
+ protected String getColumnType(Model model, ModelColumn column) {
+ final Class<?> type = column.getType();
if (Integer.class.equals(type) || int.class.equals(type)) {
return "INT";
} else if (Long.class.equals(type) || long.class.equals(type)) {
@@ -105,16 +106,11 @@ public class H2TableInstaller extends ModelInstaller {
} else if (Double.class.equals(type) || double.class.equals(type)) {
return "DOUBLE";
} else if (String.class.equals(type)) {
- return "VARCHAR(2000)";
+ return "VARCHAR(" + column.getLength() + ")";
} else if (IntKeyLongValueHashMap.class.equals(type)) {
- return "VARCHAR(20000)";
+ return "MEDIUMTEXT";
} else if (byte[].class.equals(type)) {
- if (DefaultScopeDefine.SEGMENT == model.getScopeId()) {
- if (name.getName().equals(SegmentRecord.DATA_BINARY)) {
- return "MEDIUMTEXT";
- }
- }
- return "VARCHAR(20000)";
+ return "MEDIUMTEXT";
} else {
throw new IllegalArgumentException("Unsupported data type: " + type.getName());
}
diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java
index f0320c2..f69eb07 100644
--- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java
+++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTableInstaller.java
@@ -23,13 +23,11 @@ import java.sql.SQLException;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.core.alarm.AlarmRecord;
import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord;
-import org.apache.skywalking.oap.server.core.analysis.metrics.IntKeyLongValueHashMap;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.profile.ProfileTaskLogRecord;
import org.apache.skywalking.oap.server.core.profile.ProfileThreadSnapshotRecord;
import org.apache.skywalking.oap.server.core.register.RegisterSource;
import org.apache.skywalking.oap.server.core.storage.StorageException;
-import org.apache.skywalking.oap.server.core.storage.model.ColumnName;
import org.apache.skywalking.oap.server.core.storage.model.Model;
import org.apache.skywalking.oap.server.library.client.Client;
import org.apache.skywalking.oap.server.library.client.jdbc.JDBCClientException;
@@ -68,43 +66,6 @@ public class MySQLTableInstaller extends H2TableInstaller {
}
/**
- * Based on MySQL features, provide a specific data type mappings.
- */
- @Override
- protected String getColumnType(Model model, ColumnName name, Class<?> type) {
- if (Integer.class.equals(type) || int.class.equals(type)) {
- return "INT";
- } else if (Long.class.equals(type) || long.class.equals(type)) {
- return "BIGINT";
- } else if (Double.class.equals(type) || double.class.equals(type)) {
- return "DOUBLE";
- } else if (String.class.equals(type)) {
- if (name.getName().equals(SegmentRecord.TRACE_ID) || name.getName().equals(SegmentRecord.SEGMENT_ID)) {
- return "VARCHAR(150)";
- }
- if (Metrics.ENTITY_ID.equals(name.getName())) {
- return "VARCHAR(512)";
- }
- if (SegmentRecord.ENDPOINT_NAME.equals(name.getName()) || SegmentRecord.ENDPOINT_ID.equals(
- name.getName())) {
- return "VARCHAR(200)";
- }
- if (PROFILE_TASK_LOG == model.getScopeId() || PROFILE_TASK_SEGMENT_SNAPSHOT == model.getScopeId()) {
- if (name.getName().equals(ProfileTaskLogRecord.TASK_ID)) {
- return "VARCHAR(300)";
- }
- }
- return "VARCHAR(2000)";
- } else if (IntKeyLongValueHashMap.class.equals(type)) {
- return "MEDIUMTEXT";
- } else if (byte[].class.equals(type)) {
- return "MEDIUMTEXT";
- } else {
- throw new IllegalArgumentException("Unsupported data type: " + type.getName());
- }
- }
-
- /**
* Create indexes of all tables. Due to MySQL storage is suitable for middle size use case and also compatible with
* TiDB users, Indexes are required for the UI query.
*