You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by lu...@apache.org on 2022/12/26 03:43:19 UTC
[skywalking] branch remove-unused-tags created (now 08051486c0)
This is an automated email from the ASF dual-hosted git repository.
lujiajing pushed a change to branch remove-unused-tags
in repository https://gitbox.apache.org/repos/asf/skywalking.git
at 08051486c0 remove time_bucket
This branch includes the following new commits:
new 08051486c0 remove time_bucket
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[skywalking] 01/01: remove time_bucket
Posted by lu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lujiajing pushed a commit to branch remove-unused-tags
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 08051486c0a2953fc8b61cb5dd46a7d8e1f73da8
Author: Megrez Lu <lu...@gmail.com>
AuthorDate: Mon Dec 26 11:42:56 2022 +0800
remove time_bucket
---
.../storage/plugin/banyandb/BanyanDBConverter.java | 17 +++++++++++++++++
.../plugin/banyandb/BanyanDBRecordsQueryDAO.java | 4 +---
.../storage/plugin/banyandb/MetadataRegistry.java | 22 ++++++++++++++++++----
.../banyandb/stream/BanyanDBAlarmQueryDAO.java | 2 +-
.../stream/BanyanDBEBPFProfilingDataDAO.java | 3 +--
.../stream/BanyanDBEBPFProfilingTaskDAO.java | 3 +--
.../BanyanDBProfileThreadSnapshotQueryDAO.java | 2 --
.../banyandb/stream/BanyanDBTraceQueryDAO.java | 1 -
8 files changed, 39 insertions(+), 15 deletions(-)
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBConverter.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBConverter.java
index a24de71149..de32a409a2 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBConverter.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBConverter.java
@@ -29,6 +29,9 @@ import org.apache.skywalking.banyandb.v1.client.StreamWrite;
import org.apache.skywalking.banyandb.v1.client.TagAndValue;
import org.apache.skywalking.banyandb.v1.client.grpc.exception.BanyanDBException;
import org.apache.skywalking.banyandb.v1.client.metadata.Serializable;
+import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
+import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
+import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
import org.apache.skywalking.oap.server.core.storage.type.StorageDataComplexObject;
@@ -48,6 +51,11 @@ public class BanyanDBConverter {
@Override
public Object get(String fieldName) {
+ if (fieldName.equals(Record.TIME_BUCKET)) {
+ final String timestampColumnName = schema.getTimestampColumn4Stream();
+ long timestampMillis = ((Number) this.get(timestampColumnName)).longValue();
+ return TimeBucket.getTimeBucket(timestampMillis, schema.getMetadata().getDownSampling());
+ }
MetadataRegistry.ColumnSpec spec = schema.getSpec(fieldName);
if (double.class.equals(spec.getColumnClass())) {
return ByteUtil.bytes2Double(rowEntity.getTagValue(fieldName));
@@ -70,6 +78,9 @@ public class BanyanDBConverter {
@Override
public void accept(String fieldName, Object fieldValue) {
+ if (fieldName.equals(Record.TIME_BUCKET)) {
+ return;
+ }
if (fieldName.equals(this.schema.getTimestampColumn4Stream())) {
streamWrite.setTimestamp((long) fieldValue);
}
@@ -125,6 +136,9 @@ public class BanyanDBConverter {
@Override
public void accept(String fieldName, Object fieldValue) {
+ if (fieldName.equals(Metrics.TIME_BUCKET)) {
+ return;
+ }
MetadataRegistry.ColumnSpec columnSpec = this.schema.getSpec(fieldName);
if (columnSpec == null) {
throw new IllegalArgumentException("fail to find tag/field[" + fieldName + "]");
@@ -229,6 +243,9 @@ public class BanyanDBConverter {
@Override
public Object get(String fieldName) {
+ if (fieldName.equals(Metrics.TIME_BUCKET)) {
+ return TimeBucket.getTimeBucket(dataPoint.getTimestamp(), schema.getMetadata().getDownSampling());
+ }
MetadataRegistry.ColumnSpec spec = schema.getSpec(fieldName);
switch (spec.getColumnType()) {
case TAG:
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBRecordsQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBRecordsQueryDAO.java
index d099a0a419..3efad49a3f 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBRecordsQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBRecordsQueryDAO.java
@@ -40,7 +40,7 @@ import java.util.List;
import java.util.Set;
public class BanyanDBRecordsQueryDAO extends AbstractBanyanDBDAO implements IRecordsQueryDAO {
- private static final Set<String> TAGS = ImmutableSet.of(TopN.TIME_BUCKET, TopN.ENTITY_ID, TopN.STATEMENT, TopN.TRACE_ID);
+ private static final Set<String> TAGS = ImmutableSet.of(TopN.ENTITY_ID, TopN.STATEMENT, TopN.TRACE_ID);
public BanyanDBRecordsQueryDAO(BanyanDBStorageClient client) {
super(client);
@@ -62,8 +62,6 @@ public class BanyanDBRecordsQueryDAO extends AbstractBanyanDBDAO implements IRec
} else {
query.bottomN(condition.getTopN(), valueColumnName);
}
- query.and(gte(TopN.TIME_BUCKET, duration.getStartTimeBucketInSec()));
- query.and(lte(TopN.TIME_BUCKET, duration.getEndTimeBucketInSec()));
}
});
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java
index b6b2495df9..72438991ba 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/MetadataRegistry.java
@@ -60,6 +60,8 @@ import org.apache.skywalking.banyandb.v1.client.metadata.Stream;
import org.apache.skywalking.banyandb.v1.client.metadata.TagFamilySpec;
import org.apache.skywalking.oap.server.core.analysis.DownSampling;
import org.apache.skywalking.oap.server.core.analysis.metrics.IntList;
+import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
+import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.config.ConfigService;
import org.apache.skywalking.oap.server.core.query.enumeration.Step;
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
@@ -288,12 +290,18 @@ public enum MetadataRegistry {
* Parse tags' metadata for {@link Stream}
* Every field of a class is registered as a {@link org.apache.skywalking.banyandb.model.v1.BanyandbModel.Tag}
* regardless of its dataType.
+ *
+ * @since 9.4.0 Skip {@link Record#TIME_BUCKET}
*/
List<TagMetadata> parseTagMetadata(Model model, Schema.SchemaBuilder builder) {
List<TagMetadata> tagMetadataList = new ArrayList<>();
for (final ModelColumn col : model.getColumns()) {
+ final String columnStorageName = col.getColumnName().getStorageName();
+ if (columnStorageName.equals(Record.TIME_BUCKET)) {
+ continue;
+ }
final TagFamilySpec.TagSpec tagSpec = parseTagSpec(col);
- builder.spec(col.getColumnName().getStorageName(), new ColumnSpec(ColumnType.TAG, col.getType()));
+ builder.spec(columnStorageName, new ColumnSpec(ColumnType.TAG, col.getType()));
if (col.shouldIndex()) {
// build indexRule
IndexRule indexRule = parseIndexRule(tagSpec.getTagName(), col);
@@ -310,6 +318,8 @@ public enum MetadataRegistry {
* Parse tags and fields' metadata for {@link Measure}.
* For field whose dataType is not {@link Column.ValueDataType#NOT_VALUE},
* it is registered as {@link org.apache.skywalking.banyandb.measure.v1.BanyandbMeasure.DataPoint.Field}
+ *
+ * @since 9.4.0 Skip {@link Metrics#TIME_BUCKET}
*/
List<TagMetadata> parseTagAndFieldMetadata(Model model, Schema.SchemaBuilder builder) {
List<TagMetadata> tagMetadataList = new ArrayList<>();
@@ -317,12 +327,16 @@ public enum MetadataRegistry {
Optional<ValueColumnMetadata.ValueColumn> valueColumnOpt = ValueColumnMetadata.INSTANCE
.readValueColumnDefinition(model.getName());
for (final ModelColumn col : model.getColumns()) {
- if (valueColumnOpt.isPresent() && valueColumnOpt.get().getValueCName().equals(col.getColumnName().getStorageName())) {
- builder.spec(col.getColumnName().getStorageName(), new ColumnSpec(ColumnType.FIELD, col.getType()));
+ final String columnStorageName = col.getColumnName().getStorageName();
+ if (columnStorageName.equals(Metrics.TIME_BUCKET)) {
+ continue;
+ }
+ if (valueColumnOpt.isPresent() && valueColumnOpt.get().getValueCName().equals(columnStorageName)) {
+ builder.spec(columnStorageName, new ColumnSpec(ColumnType.FIELD, col.getType()));
continue;
}
final TagFamilySpec.TagSpec tagSpec = parseTagSpec(col);
- builder.spec(col.getColumnName().getStorageName(), new ColumnSpec(ColumnType.TAG, col.getType()));
+ builder.spec(columnStorageName, new ColumnSpec(ColumnType.TAG, col.getType()));
if (col.shouldIndex()) {
// build indexRule
IndexRule indexRule = parseIndexRule(tagSpec.getTagName(), col);
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBAlarmQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBAlarmQueryDAO.java
index 812f6f84af..5a04c044e0 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBAlarmQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBAlarmQueryDAO.java
@@ -48,7 +48,7 @@ import java.util.Set;
public class BanyanDBAlarmQueryDAO extends AbstractBanyanDBDAO implements IAlarmQueryDAO {
private static final Set<String> TAGS = ImmutableSet.of(AlarmRecord.SCOPE,
AlarmRecord.NAME, AlarmRecord.ID0, AlarmRecord.ID1, AlarmRecord.ALARM_MESSAGE, AlarmRecord.START_TIME,
- AlarmRecord.TIME_BUCKET, AlarmRecord.RULE_NAME, AlarmRecord.TAGS, AlarmRecord.TAGS_RAW_DATA);
+ AlarmRecord.RULE_NAME, AlarmRecord.TAGS, AlarmRecord.TAGS_RAW_DATA);
public BanyanDBAlarmQueryDAO(BanyanDBStorageClient client) {
super(client);
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBEBPFProfilingDataDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBEBPFProfilingDataDAO.java
index 0b1942d23e..c03a4ff3be 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBEBPFProfilingDataDAO.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBEBPFProfilingDataDAO.java
@@ -39,8 +39,7 @@ public class BanyanDBEBPFProfilingDataDAO extends AbstractBanyanDBDAO implements
EBPFProfilingDataRecord.STACK_ID_LIST,
EBPFProfilingDataRecord.TARGET_TYPE,
EBPFProfilingDataRecord.DATA_BINARY,
- EBPFProfilingDataRecord.TASK_ID,
- EBPFProfilingDataRecord.TIME_BUCKET);
+ EBPFProfilingDataRecord.TASK_ID);
public BanyanDBEBPFProfilingDataDAO(BanyanDBStorageClient client) {
super(client);
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBEBPFProfilingTaskDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBEBPFProfilingTaskDAO.java
index b024bcece6..acd5fa8e07 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBEBPFProfilingTaskDAO.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBEBPFProfilingTaskDAO.java
@@ -54,8 +54,7 @@ public class BanyanDBEBPFProfilingTaskDAO extends AbstractBanyanDBDAO implements
EBPFProfilingTaskRecord.FIXED_TRIGGER_DURATION,
EBPFProfilingTaskRecord.TARGET_TYPE,
EBPFProfilingTaskRecord.CREATE_TIME,
- EBPFProfilingTaskRecord.LAST_UPDATE_TIME,
- EBPFProfilingTaskRecord.TIME_BUCKET);
+ EBPFProfilingTaskRecord.LAST_UPDATE_TIME);
private static final Gson GSON = new Gson();
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileThreadSnapshotQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileThreadSnapshotQueryDAO.java
index 985544bbe9..fe4b815122 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileThreadSnapshotQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileThreadSnapshotQueryDAO.java
@@ -49,7 +49,6 @@ public class BanyanDBProfileThreadSnapshotQueryDAO extends AbstractBanyanDBDAO i
ProfileThreadSnapshotRecord.SEGMENT_ID,
ProfileThreadSnapshotRecord.DUMP_TIME,
ProfileThreadSnapshotRecord.SEQUENCE,
- ProfileThreadSnapshotRecord.TIME_BUCKET,
ProfileThreadSnapshotRecord.STACK_BINARY);
private static final Set<String> TAGS_TRACE = ImmutableSet.of(SegmentRecord.TRACE_ID,
@@ -67,7 +66,6 @@ public class BanyanDBProfileThreadSnapshotQueryDAO extends AbstractBanyanDBDAO i
SegmentRecord.ENDPOINT_ID,
SegmentRecord.LATENCY,
SegmentRecord.START_TIME,
- SegmentRecord.TIME_BUCKET,
SegmentRecord.DATA_BINARY);
private final int querySegmentMaxSize;
diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBTraceQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBTraceQueryDAO.java
index 5b6540a244..f6491cd43d 100644
--- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBTraceQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBTraceQueryDAO.java
@@ -67,7 +67,6 @@ public class BanyanDBTraceQueryDAO extends AbstractBanyanDBDAO implements ITrace
SegmentRecord.ENDPOINT_ID,
SegmentRecord.LATENCY,
SegmentRecord.START_TIME,
- SegmentRecord.TIME_BUCKET,
SegmentRecord.DATA_BINARY);
public BanyanDBTraceQueryDAO(BanyanDBStorageClient client) {