You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2025/08/27 23:50:10 UTC
(skywalking-banyandb) 19/26: Refactor user keys handling: Update the processing of user-provided int64 keys to write encoded data directly without compression. Adjust related comments and remove decompression steps in the readAll and loadBlockData methods for improved efficiency and clarity.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch sidx/query
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 38d88711f089a05d4acc9d9a0d82f6ef03baa47c
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed Aug 27 08:59:55 2025 +0800
Refactor user keys handling: Update the processing of user-provided int64 keys to write encoded data directly without compression. Adjust related comments and remove decompression steps in the readAll and loadBlockData methods for improved efficiency and clarity.
---
banyand/internal/sidx/block.go | 7 +++----
banyand/internal/sidx/part.go | 16 +++-------------
banyand/internal/sidx/query_result.go | 12 ++++--------
3 files changed, 10 insertions(+), 25 deletions(-)
diff --git a/banyand/internal/sidx/block.go b/banyand/internal/sidx/block.go
index cc638542..24be89d9 100644
--- a/banyand/internal/sidx/block.go
+++ b/banyand/internal/sidx/block.go
@@ -335,11 +335,10 @@ func mustWriteKeysTo(kb *dataBlock, userKeys []int64, keysWriter *writer) (encod
var firstValue int64
bb.Buf, encodeType, firstValue = encoding.Int64ListToBytes(bb.Buf[:0], userKeys)
- // Compress and write
- compressedData := zstd.Compress(nil, bb.Buf, 1)
+ // Write encoded data directly without compression
kb.offset = keysWriter.bytesWritten
- kb.size = uint64(len(compressedData))
- keysWriter.MustWrite(compressedData)
+ kb.size = uint64(len(bb.Buf))
+ keysWriter.MustWrite(bb.Buf)
return encodeType, firstValue
}
diff --git a/banyand/internal/sidx/part.go b/banyand/internal/sidx/part.go
index 7aa6805d..5753a300 100644
--- a/banyand/internal/sidx/part.go
+++ b/banyand/internal/sidx/part.go
@@ -52,7 +52,7 @@ const (
// Each part contains multiple files organized by type:
// - primary.bin: Block metadata and structure information
// - data.bin: User payload data (compressed)
-// - keys.bin: User-provided int64 keys (compressed)
+// - keys.bin: User-provided int64 keys (encoded but not compressed)
// - meta.bin: Part metadata
// - <name>.td: Tag data files (one per tag)
// - <name>.tm: Tag metadata files (one per tag)
@@ -272,7 +272,6 @@ func (p *part) readAll() ([]*elements, error) {
compressedDataBuf := make([]byte, 0, 1024)
dataBuf := make([]byte, 0, 1024)
compressedKeysBuf := make([]byte, 0, 1024)
- keysBuf := make([]byte, 0, 1024)
for _, pbm := range p.primaryBlockMetadata {
// Read and decompress primary block metadata
@@ -310,17 +309,8 @@ func (p *part) readAll() ([]*elements, error) {
compressedKeysBuf = bytes.ResizeOver(compressedKeysBuf, int(bm.keysBlock.size))
fs.MustReadData(p.keys, int64(bm.keysBlock.offset), compressedKeysBuf)
- keysBuf, err = zstd.Decompress(keysBuf[:0], compressedKeysBuf)
- if err != nil {
- releaseElements(elems)
- for _, e := range result {
- releaseElements(e)
- }
- return nil, fmt.Errorf("cannot decompress keys block: %w", err)
- }
-
- // Decode user keys using the stored encoding information
- elems.userKeys, err = encoding.BytesToInt64List(elems.userKeys[:0], keysBuf, bm.keysEncodeType, bm.minKey, int(bm.count))
+ // Decode user keys directly using the stored encoding information
+ elems.userKeys, err = encoding.BytesToInt64List(elems.userKeys[:0], compressedKeysBuf, bm.keysEncodeType, bm.minKey, int(bm.count))
if err != nil {
releaseElements(elems)
for _, e := range result {
diff --git a/banyand/internal/sidx/query_result.go b/banyand/internal/sidx/query_result.go
index d7789428..dfe07d80 100644
--- a/banyand/internal/sidx/query_result.go
+++ b/banyand/internal/sidx/query_result.go
@@ -178,7 +178,7 @@ func (qr *queryResult) loadBlockData(tmpBlock *block, p *part, bm *blockMetadata
return false
}
- // Read and decompress user keys (always needed)
+ // Read user keys (always needed)
bb := bigValuePool.Get()
if bb == nil {
bb = &bytes.Buffer{}
@@ -191,13 +191,9 @@ func (qr *queryResult) loadBlockData(tmpBlock *block, p *part, bm *blockMetadata
bb.Buf = bytes.ResizeOver(bb.Buf[:0], int(bm.keysBlock.size))
fs.MustReadData(p.keys, int64(bm.keysBlock.offset), bb.Buf)
- keysBuf, err := zstd.Decompress(nil, bb.Buf)
- if err != nil {
- return false
- }
-
- // Decode user keys
- tmpBlock.userKeys, err = encoding.BytesToInt64List(tmpBlock.userKeys[:0], keysBuf, bm.keysEncodeType, bm.minKey, int(bm.count))
+ // Decode user keys directly
+ var err error
+ tmpBlock.userKeys, err = encoding.BytesToInt64List(tmpBlock.userKeys[:0], bb.Buf, bm.keysEncodeType, bm.minKey, int(bm.count))
if err != nil {
return false
}