(skywalking-banyandb) 01/01: Add GranularTTL message for fine-grained TTL control and update ResourceOpts to include granular_ttl field

[ha...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

hanahmily pushed a commit to branch api/tag-field-ttl
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git

commit 50fea01df4c3764792d1660efaadbc31c888d48c
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Thu Nov 27 10:30:25 2025 +0000

    Add GranularTTL message for fine-grained TTL control and update ResourceOpts to include granular_ttl field
---
 api/proto/banyandb/common/v1/common.proto | 51 +++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/api/proto/banyandb/common/v1/common.proto b/api/proto/banyandb/common/v1/common.proto
index 644b08de..5b90c552 100644
--- a/api/proto/banyandb/common/v1/common.proto
+++ b/api/proto/banyandb/common/v1/common.proto
@@ -91,6 +91,53 @@ message LifecycleStage {
   uint32 replicas = 7;
 }
 
+// TagList represents a list of tag names within a tag family.
+message TagList {
+  repeated string tags = 1;
+}
+
+// GranularTTL defines TTL rules for specific tag families, tags, or fields.
+// This allows fine-grained control over data retention at a more granular level than Group TTL.
+//
+// Matching Logic:
+// For a given column (Tag Family TF, Tag T or Field F) in a Subject S:
+// 1. Iterate through granular_ttl rules in order.
+// 2. For each rule:
+//    - Check Subject: S must match subject_include (empty list matches ALL subjects) 
+//      and NOT match subject_exclude. subject_exclude takes precedence.
+//    - Check Tag Family: TF must be present in tag_family_include keys.
+//      - If present, check Tag T: T must be in the TagList value (empty list or "*" matches ALL tags).
+//      - Check Exclude: TF must NOT be in tag_family_exclude keys 
+//        (or if present, T must NOT be in its TagList value).
+//    - Check Field: F must match field_include (empty list matches NO fields, "*" matches ALL)
+//      and NOT match field_exclude.
+// 3. The first matching rule determines the TTL for that column.
+// 4. If no rule matches, the column retains the default behavior (Group TTL).
+message GranularTTL {
+  // The TTL duration for the matched targets.
+  // Note: This cannot exceed the Group's main TTL (ResourceOpts.ttl).
+  // If set longer, the data will still be removed when the segment is expired by the Group TTL.
+  IntervalRule ttl = 1 [(validate.rules).message.required = true];
+
+  // Subject filters (Stream/Measure/Trace names).
+  // If subject_include is empty, it matches ALL subjects in the Group.
+  // subject_exclude takes precedence over subject_include.
+  repeated string subject_include = 2;
+  repeated string subject_exclude = 3;
+
+  // Tag Family filters.
+  // The key is the tag family name. The value is a list of tags in the tag family.
+  // If the value list is empty, it matches ALL tags in the tag family.
+  // If the value list contains "*", it matches ALL tags in the tag family.
+  map<string, TagList> tag_family_include = 4;
+  map<string, TagList> tag_family_exclude = 5;
+
+  // Field filters.
+  // If field_include is empty, NO fields are matched (unless "*" is used).
+  repeated string field_include = 6;
+  repeated string field_exclude = 7;
+}
+
 message ResourceOpts {
   // shard_num is the number of shards
   uint32 shard_num = 1 [(validate.rules).uint32.gt = 0];
@@ -107,6 +154,10 @@ message ResourceOpts {
   // A value of 0 means no replicas, while a value of 1 means one primary shard and one replica.
   // Higher values indicate more replicas.
   uint32 replicas = 6;
+  // granular_ttl defines a list of TTL rules for specific tag families or fields.
+  // Rules are evaluated in order. The first rule that matches a specific column (TagFamily or Field) applies.
+  // See GranularTTL message for detailed matching logic.
+  repeated GranularTTL granular_ttl = 7;
 }
 
 // Group is an internal object for Group management