You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2025/11/27 10:41:00 UTC
(skywalking-banyandb) 01/01: Add GranularTTL message for fine-grained TTL control
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch api/tag-field-ttl
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 4f2e3b9b18714b76328b9c4f06c7a9a3d6c68639
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Thu Nov 27 10:30:25 2025 +0000
Add GranularTTL message for fine-grained TTL control
---
api/proto/banyandb/common/v1/common.proto | 52 +++++++++++++++++++++++++++++++
1 file changed, 52 insertions(+)
diff --git a/api/proto/banyandb/common/v1/common.proto b/api/proto/banyandb/common/v1/common.proto
index 644b08de..c5a25476 100644
--- a/api/proto/banyandb/common/v1/common.proto
+++ b/api/proto/banyandb/common/v1/common.proto
@@ -89,8 +89,60 @@ message LifecycleStage {
// A value of 0 means no replicas, while a value of 1 means one primary shard and one replica.
// Higher values indicate more replicas.
uint32 replicas = 7;
+ // granular_ttl defines TTL rules for specific tag families, tags, or fields.
+ // These rules are applied when data is migrated TO this stage.
+ // The TTL duration used for expiration check comes from this stage's ttl field.
+ // See GranularTTL message for detailed matching logic.
+ repeated GranularTTL granular_ttl = 8;
}
+// TagList represents a list of tag names within a tag family.
+message TagList {
+ repeated string tags = 1;
+}
+
+// GranularTTL defines TTL rules for specific tag families, tags, or fields.
+// This allows fine-grained control over data retention at a more granular level than stage TTL.
+// These rules are scoped to the LifecycleStage that contains them.
+// The TTL duration used for expiration check comes from the parent LifecycleStage's ttl field.
+//
+// Matching Logic:
+// For a given column (Tag Family TF, Tag T or Field F) in a Subject S when migrating TO a stage:
+// 1. Get the granular_ttl rules from the target LifecycleStage.
+// 2. Iterate through these rules in order.
+// 3. For each rule:
+// - Check Subject: S must match subject_include (empty list matches ALL subjects)
+// and NOT match subject_exclude. subject_exclude takes precedence.
+// - Check Tag Family: TF must be present in tag_family_include keys.
+// - If present, check Tag T: T must be in the TagList value (empty list or "*" matches ALL tags).
+// - Check Exclude: TF must NOT be in tag_family_exclude keys
+// (or if present, T must NOT be in its TagList value).
+// - Check Field: F must match field_include (empty list matches NO fields, "*" matches ALL)
+// and NOT match field_exclude.
+// 4. The first matching rule determines that the column should be filtered when migrating to that stage.
+// 5. If no rule matches, the column retains the default behavior (no filtering, full migration).
+// 6. The TTL duration used for expiration check comes from the parent LifecycleStage.ttl.
+message GranularTTL {
+ // Subject filters (Stream/Measure/Trace names).
+ // If subject_include is empty, it matches ALL subjects in the Group.
+ // subject_exclude takes precedence over subject_include.
+ repeated string subject_include = 1;
+ repeated string subject_exclude = 2;
+
+ // Tag Family filters.
+ // The key is the tag family name. The value is a list of tags in the tag family.
+ // If the value list is empty, it matches ALL tags in the tag family.
+ // If the value list contains "*", it matches ALL tags in the tag family.
+ map<string, TagList> tag_family_include = 3;
+ map<string, TagList> tag_family_exclude = 4;
+
+ // Field filters.
+ // If field_include is empty, NO fields are matched (unless "*" is used).
+ repeated string field_include = 5;
+ repeated string field_exclude = 6;
+}
+
+
message ResourceOpts {
// shard_num is the number of shards
uint32 shard_num = 1 [(validate.rules).uint32.gt = 0];