You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2026/05/08 08:39:15 UTC
(skywalking-banyandb) branch v0.10.x updated (55c6007c2 -> c9961ab39)
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a change to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
from 55c6007c2 chore: update CHANGES.md and sync .gitignore from main
new 7f7a5683b Stable Segment Endtime (#1051)
new 18477053b fix(test): merge query_ondisk into query and fix flaky on-disk integration tests (#1041)
new 7ff0d5828 fix(ui): correct query editor refresh/reset behavior and BydbQL keyword highlighting (#1045)
new b98ee96bc Fix expired TLS test certificate (#1078)
new ce600a702 fix(measure,stream,trace): wait until all mem parts flushed in file_snapshot tests (#1098)
new 628f5def1 fix(trace): resolve flaky schema change merge test (#1099)
new 935db739e fix: stabilize flaky TestCollectWithPartialClosedSegments and fit flaky-test in 50min (#1100)
new 5cfe59a12 Fix FODC lifecycle endpoint wrong encoding and increase the gRPC timeout between agent and proxy (#1103)
new 468cb5d44 fix(trace): accept out-of-order timestamps within the same traceID (#1114)
new 6ce814162 Fix fetch segment nil pointer in cold stage and propagation the error when inspect the group (#1115)
new e4e10d309 Fix missing property group adapt when inspect group (#1117)
new af88eb865 fix(banyand): close merge durability gap (#13862) (#1118)
new 1f8e29f37 chore: fix lint issues after cherry-pick
new c9961ab39 Support multiple types for the same tag in a single trace or sidx part (#1066)
The 14 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/flaky-test.yml | 2 +-
CHANGES.md | 86 +++-
api/proto/banyandb/database/v1/rpc.proto | 2 +
api/proto/banyandb/fodc/v1/rpc.proto | 11 +
api/validate/validate.go | 26 +
banyand/internal/sidx/block.go | 2 +
banyand/internal/sidx/interfaces.go | 38 +-
banyand/internal/sidx/merge.go | 87 +++-
banyand/internal/sidx/merge_test.go | 121 +++++
banyand/internal/sidx/merger_durability_test.go | 95 ++++
banyand/internal/sidx/metadata.go | 2 +-
banyand/internal/sidx/part.go | 19 +-
banyand/internal/sidx/query_result.go | 26 +-
banyand/internal/sidx/scan_query.go | 17 +-
banyand/internal/sidx/tag.go | 79 ++-
banyand/internal/storage/index.go | 49 +-
banyand/internal/storage/segment.go | 80 ++-
banyand/internal/storage/segment_test.go | 537 +++++++++++++++++++++
banyand/internal/storage/tsdb_test.go | 4 +-
banyand/internal/storage/version.go | 23 +-
banyand/internal/storage/version_test.go | 63 +++
banyand/internal/storage/versions.yml | 1 +
banyand/liaison/grpc/deletion.go | 14 +-
banyand/liaison/grpc/deletion_test.go | 28 +-
banyand/liaison/grpc/registry.go | 9 +-
banyand/measure/merger.go | 3 +-
banyand/measure/merger_durability_test.go | 82 ++++
banyand/measure/merger_test.go | 4 +-
banyand/measure/metadata.go | 3 +-
banyand/measure/metadata_internal_test.go | 48 ++
banyand/measure/part.go | 13 +-
banyand/measure/part_metadata.go | 2 +-
banyand/measure/query_test.go | 64 +--
banyand/measure/tstable.go | 27 +-
banyand/measure/write_data.go | 3 +-
banyand/metadata/client.go | 2 +-
banyand/metadata/metadata.go | 2 +-
banyand/metadata/schema/collector.go | 64 ++-
banyand/metadata/schema/collector_test.go | 118 +++++
.../schema/property/testdata/certs/server.crt | 47 +-
banyand/queue/sub/group_lifecycle.go | 14 +-
.../queue/sub/group_lifecycle_cold_tier_test.go | 298 ++++++++++++
banyand/queue/sub/group_lifecycle_test.go | 116 ++++-
banyand/stream/block_scanner_test.go | 42 +-
banyand/stream/merger.go | 3 +-
banyand/stream/merger_durability_test.go | 80 +++
banyand/stream/metadata.go | 4 +-
banyand/stream/metadata_internal_test.go | 48 ++
banyand/stream/part.go | 13 +-
banyand/stream/part_metadata.go | 2 +-
banyand/stream/query_by_idx_test.go | 22 +-
banyand/stream/tstable.go | 27 +-
banyand/stream/write_data.go | 3 +-
banyand/trace/block.go | 31 +-
banyand/trace/block_writer.go | 36 +-
banyand/trace/block_writer_test.go | 50 ++
banyand/trace/merger.go | 88 +++-
banyand/trace/merger_bench_test.go | 8 +-
banyand/trace/merger_durability_test.go | 179 +++++++
banyand/trace/merger_test.go | 154 +++++-
banyand/trace/metadata.go | 4 +-
banyand/trace/metadata_internal_test.go | 48 ++
banyand/trace/metadata_test.go | 91 +++-
banyand/trace/part.go | 13 +-
banyand/trace/part_metadata.go | 6 +-
banyand/trace/query.go | 15 +-
banyand/trace/query_test.go | 42 +-
banyand/trace/svc_standalone.go | 1 +
banyand/trace/tag.go | 53 ++
banyand/trace/tag_test.go | 19 +
banyand/trace/trace_suite_test.go | 2 +-
banyand/trace/tstable.go | 27 +-
banyand/trace/write_data.go | 7 +-
dist/LICENSE | 4 +-
docs/api-reference.md | 2 +
fodc/agent/internal/lifecycle/collector.go | 56 ++-
fodc/agent/internal/lifecycle/collector_test.go | 263 +++++++++-
.../internal/timeouts/timeouts.go | 50 +-
fodc/proxy/internal/api/server.go | 66 ++-
fodc/proxy/internal/api/server_test.go | 196 ++++++++
fodc/proxy/internal/lifecycle/manager.go | 37 +-
fodc/proxy/internal/lifecycle/manager_test.go | 61 +++
pkg/fs/file_system.go | 59 +++
pkg/fs/local_file_system.go | 99 +++-
pkg/fs/local_file_system_darwin.go | 17 +
pkg/fs/local_file_system_linux.go | 17 +
pkg/fs/local_file_system_test.go | 62 +++
.../local_file_system_testhook_test.go} | 19 +-
pkg/fs/local_file_system_windows.go | 6 +
pkg/timestamp/scheduler.go | 8 +-
test/cases/measure/measure.go | 11 +-
test/cases/property/property.go | 11 +-
test/cases/stream/stream.go | 19 +-
test/cases/topn/topn.go | 5 -
test/cases/trace/trace.go | 19 +-
.../standalone/multi_segments/common.go | 4 +
.../standalone/multi_segments/etcd/suite_test.go | 24 +-
.../multi_segments/property/suite_test.go | 21 +-
.../standalone/multi_segments/round2.go | 61 +++
test/integration/standalone/query/common.go | 4 +
.../standalone/query/etcd/suite_test.go | 28 +-
.../standalone/query/property/suite_test.go | 21 +-
test/integration/standalone/query/round2.go | 64 +++
test/integration/standalone/query_ondisk/common.go | 103 ----
.../standalone/query_ondisk/etcd/suite_test.go | 64 ---
.../standalone/query_ondisk/property/suite_test.go | 69 ---
test_table | Bin 0 -> 2254768 bytes
ui/LICENSE | 4 +-
ui/package-lock.json | 417 ++++++++--------
ui/src/components/CodeMirror/bydbql-mode.js | 9 +
ui/src/components/Property/PropertyRead.vue | 36 +-
ui/src/components/Read/index.vue | 14 +-
112 files changed, 4526 insertions(+), 953 deletions(-)
create mode 100644 banyand/internal/sidx/merger_durability_test.go
create mode 100644 banyand/internal/storage/version_test.go
create mode 100644 banyand/measure/merger_durability_test.go
create mode 100644 banyand/measure/metadata_internal_test.go
create mode 100644 banyand/metadata/schema/collector_test.go
create mode 100644 banyand/queue/sub/group_lifecycle_cold_tier_test.go
create mode 100644 banyand/stream/merger_durability_test.go
create mode 100644 banyand/stream/metadata_internal_test.go
create mode 100644 banyand/trace/merger_durability_test.go
create mode 100644 banyand/trace/metadata_internal_test.go
copy api/proto/banyandb/trace/v1/write.proto => fodc/internal/timeouts/timeouts.go (50%)
copy pkg/{convert/hash.go => fs/local_file_system_testhook_test.go} (65%)
create mode 100644 test/integration/standalone/multi_segments/round2.go
create mode 100644 test/integration/standalone/query/round2.go
delete mode 100644 test/integration/standalone/query_ondisk/common.go
delete mode 100644 test/integration/standalone/query_ondisk/etcd/suite_test.go
delete mode 100644 test/integration/standalone/query_ondisk/property/suite_test.go
create mode 100755 test_table
(skywalking-banyandb) 12/14: fix(banyand): close merge durability gap (#13862) (#1118)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit af88eb86594d3841cedaa09c617cb7a0cb9398ad
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Fri May 8 10:01:55 2026 +0800
fix(banyand): close merge durability gap (#13862) (#1118)
* fix(fs): propagate fdatasync error in seqWriter.Close
The previous implementation discarded the SyncAndDropCache return value
(`_ = SyncAndDropCache(...)`), so a fdatasync failure (ENOSPC, EIO,
EBADF) silently lost data. The merge-write path used by trace, measure,
stream, and sidx all flow through seqWriter.Close, so the silent loss
applied to every part written to disk.
Now the error is checked and returned wrapped in a FileSystemError. The
existing bw.writers.MustClose() callers in each engine continue to
panic on error — that turns silent data loss on disk failure into a
visible panic operators can act on, consistent with BanyanDB's
fail-fast contract.
The skipFadvise=true branch falls back to file.Sync() with the same
error propagation, so the no-cache path is now also durable.
Note on ownership: seqWriter borrows the *os.File from LocalFile and
does not own it; the fd is closed by LocalFile.Close(). seqWriter.Close
intentionally does NOT close the fd.
Part of apache/skywalking#13862.
* fix(fs): fsync metadata file in localFileSystem.Write
metadata.json, traceID.filter, tag.type, manifest.json (sidx), and
similar atomic state files all flow through localFileSystem.Write,
which previously did write+close with no fsync between. A crash after
the parent dir was fsynced (so the directory entry was on disk) but
before the inode's data was flushed left a metadata.json with zero or
partial content even though the merger thought the write had completed.
Add file.Sync() between file.Write and the deferred file.Close so the
bytes hit disk before Write returns. The error from Sync is propagated
as flushError to match the existing failure-mode style.
Part of apache/skywalking#13862.
* feat(fs): add WriteAtomic, MustFlushAtomic, and CleanupLeftoverTmp
WriteAtomic does write-tmp -> file.Sync(tmp) -> rename -> dir.Sync().
After a successful return the content is durable on a POSIX-compliant
local filesystem. A crash between fsync(tmp) and rename leaves only the
.tmp; a crash after rename leaves a healthy final.
MustFlushAtomic mirrors MustFlush's panic-on-error wrapper for callers
that already use the panic style (sidx).
CleanupLeftoverTmp removes <file>.tmp siblings of healthy <file> finals
on part open — safe post-rename leftover from a re-tried merge. A .tmp
without its final is left alone so the engine's existing
mustReadMetadata panic fires for operator intervention, per BanyanDB's
canonical fail-fast contract.
testHookAfterTmpFsync lives in pkg/fs/local_file_system_testhook_test.go
(_test.go suffix) so production binaries do not carry the indirection.
Engines (trace, measure, stream, sidx) adopt these in follow-up commits.
Part of apache/skywalking#13862.
* fix(trace): two-phase commit for part metadata
Trace's three metadata writers — mustWriteMetadata (metadata.json),
mustWriteTagType (tag.type), and mustWriteTraceIDFilter (traceID.filter)
— now use fileSystem.WriteAtomic instead of fileSystem.Write. A clean
kernel crash mid-flush or mid-merge can no longer leave a part dir where
the metadata files claim bytes the data files do not contain: the
rename is atomic, so either the final exists (durable) or it does not
(operator-actionable, the existing mustReadMetadata panic at boot is
the canonical signal).
mustOpenFilePart calls fs.CleanupLeftoverTmp before mustReadMetadata to
remove safe post-rename leftovers (stray <file>.tmp next to a healthy
final from a re-tried merge or filesystem replay). A .tmp without its
final is left in place so the existing panic-on-missing-metadata fires
for operator intervention, per BanyanDB's canonical fail-fast pattern.
Affects all three callers of trace's metadata helpers: the flush path
(memPart.mustFlush), the merge path (mergeParts), and the streaming
sync path.
Part of apache/skywalking#13862.
* fix(measure): two-phase commit for part metadata
Same recipe as trace (apache/skywalking#13862): mustWriteMetadata at
banyand/measure/part_metadata.go now uses fileSystem.WriteAtomic, and
mustOpenFilePart at banyand/measure/part.go calls fs.CleanupLeftoverTmp
to remove safe post-rename leftovers.
A clean kernel crash mid-merge or mid-flush can no longer leave a part
dir where metadata.json claims bytes the data files do not contain.
Affects all three callers of measure's mustWriteMetadata: the flush
path (memPart.mustFlush), the merge path, and the streaming sync path.
* fix(stream): two-phase commit for part metadata
Same recipe as trace and measure (apache/skywalking#13862):
mustWriteMetadata at banyand/stream/part_metadata.go now uses
fileSystem.WriteAtomic, and mustOpenFilePart at banyand/stream/part.go
calls fs.CleanupLeftoverTmp to remove safe post-rename leftovers.
Affects all three callers of stream's mustWriteMetadata: the flush
path, the merge path, and the streaming sync path.
* fix(sidx): two-phase commit for manifest.json
Same recipe as trace, measure, and stream (apache/skywalking#13862),
adjusted for sidx's panic-on-error helper style. mustWriteMetadata at
banyand/internal/sidx/metadata.go now calls fs.MustFlushAtomic instead
of fs.MustFlush. mustOpenPart at banyand/internal/sidx/part.go calls
fs.CleanupLeftoverTmp before opening the part's files.
The legacy meta.bin fallback in loadPartMetadata is unaffected — the
cleanup helper matches <file>.tmp against <file> per-file, so a
manifest.json.tmp without a manifest.json final stays in place for
operator inspection (existing panic-on-missing-metadata still fires).
Affects flush, merge, and streaming sync paths — all funnel through
mustWriteMetadata.
* docs(changes): note merge durability fix under 0.11.0
CHANGES.md entry for the seven-commit merge durability fix landing as
apache/skywalking#13862 (root cause for #13861).
Skipping a separate cross-package integration test: each engine
(trace, measure, stream, sidx) already has merger_durability_test.go
that exercises the same on-disk shape using the engine's native
fixtures. A cross-package integration test would require exposing
testHookAfterTmpFsync outside pkg/fs, which would violate the
test-only-isolation invariant established by Task 3.
* test(trace): verify torn-part fail-fast contract is preserved
Test_merger_tornSpansBin_stillPanics is the reproducer described in
the body of apache/skywalking#13862. It forges the production-shape
on-disk state (a part directory whose metadata.json claims more bytes
than spans.bin actually contains) and feeds it to mergeParts.
Expected outcome: mergeParts panics with one of the canonical
fail-fast messages from pkg/fs (mustReadFull) — "offset must be
equal to bytesRead", "cannot read data: unexpected EOF", "cannot
decode spans", or similar.
Captured panic in this run:
"cannot read data: unexpected EOF" (from seqReader.mustReadFull)
This guards against accidental panic suppression by future refactors:
the durability fix prevents torn parts from being CREATED via a
clean-crash path, but a torn part forged on disk (rsync, manual
file copy, host disk failure, pre-fix-deployment leftovers) still
triggers BanyanDB's canonical fail-fast contract — by design.
* polish(fs,trace): address review nits on merge durability fix
Five MINOR findings from code review of the apache/skywalking#13862
fix:
- WriteAtomic: drop unreachable os.IsExist branch (O_CREATE without
O_EXCL never returns EEXIST). Hoist filepath.Dir(name) to a local
parentDir variable. Document the re-tried-after-crash overwrite
semantics in the godoc.
- CleanupLeftoverTmp: skip symlinks via os.Lstat — BanyanDB part
directories never contain symlinks; encountering one is unexpected.
Log Remove failures at debug level for post-mortem visibility
(previously errors were silently swallowed).
- Test_mustOpenFilePart_removesLeftoverTmp: parameterize over all
three trace .tmp shapes (metadata.json.tmp, traceID.filter.tmp,
tag.type.tmp) instead of only metadata.json.tmp. Skips per-case
when the fixture didn't produce the matching final.
Behavior unchanged on the success and panic paths; only edge-case
defenses and test coverage breadth.
* fix(fs,trace,measure,stream,sidx): address Copilot review findings
Three MAJOR findings from a follow-up review pass:
1. WriteAtomic no longer removes the .tmp on rename failure. At that
point the .tmp is fully written + fsynced + closed — the only
durable record of the new content — and an operator may need it
for forensic recovery. The next successful WriteAtomic to the same
name overwrites it via O_TRUNC, so no orphan accumulates. Aligns
with the project's no-deletion policy on potentially-useful
artifacts.
2. sidx loadPartMetadata no longer silently falls back to legacy
meta.bin when manifest.json is missing AND manifest.json.tmp is
present. That state means the most recent atomic commit crashed
between fsync(tmp) and rename; falling back to a stale meta.bin
would mask the incomplete-commit state. Now returns an error that
propagates to mustOpenPart's panic — canonical fail-fast.
3. Removed redundant SyncPath calls after metadata writes that go
through WriteAtomic / MustFlushAtomic. The atomic helpers already
fsync the parent directory after rename, and the trailing
SyncPath was issuing a second dir fsync of the same inode. Trace
was paying up to 4 dir fsyncs per finalize (3 metadata files +
trailing SyncPath); measure/stream/sidx paid 2. Now 1 each.
Affects mergeParts and memPart.mustFlush in trace/measure/stream,
sidx mergeParts and SyncPartContext.Finish, and the per-engine
streaming-sync FinishSync paths.
Also switches the trace streaming-sync sidecars (traceID.filter,
tag.type via write_data.go) from non-atomic fs.MustFlush to
fs.MustFlushAtomic — same crash-safety as the merger/flush paths
they parallel.
Tests all pass: pkg/fs (21 specs), trace, measure, stream, sidx.
* fix(trace,measure,stream): atomic write for seriesMetadata sidecar
The seriesMetadata.bin sidecar in each engine's memPart.mustFlush is
read at part open by mustOpenFilePart, so it has the same crash-safety
profile as metadata.json: a crash during a non-atomic fs.MustFlush could
leave a partial file that reads as corrupt and produces a mid-file
unmarshal panic instead of fail-fast at metadata read time.
Switch to fs.MustFlushAtomic to match the other metadata writers
(mustWriteMetadata, mustWriteTagType, mustWriteTraceIDFilter,
streaming-sync sidecars).
Out of scope for the original Copilot review pass but found while
auditing call sites; closes the last non-atomic metadata write site
in the part-flush path. Part of apache/skywalking#13862.
* fix(trace,measure,stream): atomic write for snapshot files
mustWriteSnapshot in trace, measure, and stream now uses
fs.MustFlushAtomic instead of the bespoke CreateLockFile + Write +
Close + Rename + SyncPath sequence. The previous code was missing
file.Sync() between Write and Close, so the snapshot's content was
not durable when the rename made it visible.
Why this matters: the orphan-part cleanup at startup
(banyand/{trace,measure,stream}/tstable.go in initTSTable) deletes any
part directory whose ID is NOT referenced by the loaded snapshot. The
cleanup logic implicitly trusts the snapshot to be authoritative — but
a non-durable snapshot can land on disk pointing at parts whose data
isn't actually durable yet, or fail to land and lose references to
healthy parts that then look orphaned.
WriteAtomic does write-tmp -> file.Sync(tmp) -> rename -> dir.Sync,
matching the same crash-safety contract now used for metadata.json,
manifest.json, traceID.filter, tag.type, and seriesMetadata.bin. A
crash leaves either the previous valid snapshot or only <name>.tmp,
never a renamed-but-not-durable snapshot.
This closes the last metadata-style write site that could undermine
the orphan-cleanup invariant. Part of apache/skywalking#13862.
* fix(fs,measure,trace): address PR #1118 review feedback
Three issues from CI lint and remote Copilot review on apache/skywalking-banyandb#1118:
1. CI lint failure: govet shadow at banyand/measure/merger_test.go:1460.
The local "dps" shadowed the package-level "dps" defined in
part_test.go:130. The shadow was latent until the new
merger_durability_test.go added another package-level reference,
which made govet's shadow check fire on the local. Renamed the local
to partDPs.
2. WriteAtomic's parent-directory fsync was not platform-aware.
os.File.Sync on a directory is a real operation on ext4/xfs/apfs,
but Windows does not expose directory fsync the way POSIX does
(the existing SyncPath on Windows is a no-op for the same reason).
Extracted the dir-sync into a per-platform syncDir helper:
- linux/darwin: open the directory, file.Sync, close, return error
- windows: no-op returning nil
WriteAtomic now calls syncDir(parentDir) instead of inlining the
open+sync+close. Cross-compiled clean on both windows and darwin.
3. File-descriptor leak in Test_merger_tornSpansBin_stillPanics.
p1 and p2 are opened via mustOpenFilePart (each opens many files)
and were never closed when the panic fired. Added p1.close() and
p2.close() to the deferred recover so the fds are released even on
the panic path.
All target tests still green (pkg/fs 21 specs, trace, measure 29.4s);
cross-compile clean on linux/darwin/windows.
---
CHANGES.md | 3 +
banyand/internal/sidx/merge.go | 3 +-
banyand/internal/sidx/merger_durability_test.go | 95 +++++++++++++
banyand/internal/sidx/metadata.go | 2 +-
banyand/internal/sidx/part.go | 19 ++-
banyand/measure/merger.go | 3 +-
banyand/measure/merger_durability_test.go | 82 +++++++++++
banyand/measure/merger_test.go | 4 +-
banyand/measure/part.go | 13 +-
banyand/measure/part_metadata.go | 2 +-
banyand/measure/tstable.go | 27 +---
banyand/measure/write_data.go | 3 +-
banyand/stream/merger.go | 3 +-
banyand/stream/merger_durability_test.go | 80 +++++++++++
banyand/stream/part.go | 13 +-
banyand/stream/part_metadata.go | 2 +-
banyand/stream/tstable.go | 27 +---
banyand/stream/write_data.go | 3 +-
banyand/trace/merger.go | 4 +-
banyand/trace/merger_durability_test.go | 179 ++++++++++++++++++++++++
banyand/trace/part.go | 13 +-
banyand/trace/part_metadata.go | 6 +-
banyand/trace/tstable.go | 27 +---
banyand/trace/write_data.go | 7 +-
pkg/fs/file_system.go | 59 ++++++++
pkg/fs/local_file_system.go | 99 ++++++++++++-
pkg/fs/local_file_system_darwin.go | 17 +++
pkg/fs/local_file_system_linux.go | 17 +++
pkg/fs/local_file_system_test.go | 62 ++++++++
pkg/fs/local_file_system_testhook_test.go | 29 ++++
pkg/fs/local_file_system_windows.go | 6 +
31 files changed, 813 insertions(+), 96 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 22ba94f29..d912f0c04 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -8,6 +8,9 @@ Release Notes.
- Fix reuse of byte arrays in min/max implementation causing data corruption.
- Fix index-mode measure queries returning documents outside requested time range.
+- Close BanyanDB merge write-path durability gap that allowed torn parts to be created by a crash between data write and metadata commit. Metadata files (`metadata.json` for trace/measure/stream, `manifest.json` for sidx, plus `traceID.filter` and `tag.type`) now go through a new `WriteAtomic` (write-tmp + fsync + rename + fsync-dir) sequence; data writers (`seqWriter.Close`, `localFileSystem.Write`) now propagate fdatasync errors instead of silently dropping them. `mustOpenFilePart` / ` [...]
+- Fix bydbctl command tests using global stdout capture, which caused race-enabled runs to corrupt captured command output.
+- Use `topic` instead of `session_id` as the Prometheus label on liaison `queue_sub` chunk-ordering counters to avoid unbounded metric cardinality.
- Fix flaky trace query filtering caused by non-deterministic sidx tag ordering and add consistency checks for integration query cases.
- MCP: Add validation for properties and harden the mcp server.
- Fix property schema client connection not stable after data node restarted.
diff --git a/banyand/internal/sidx/merge.go b/banyand/internal/sidx/merge.go
index 157709698..ac901a2b5 100644
--- a/banyand/internal/sidx/merge.go
+++ b/banyand/internal/sidx/merge.go
@@ -126,7 +126,8 @@ func (s *sidx) mergeParts(fileSystem fs.FileSystem, closeCh <-chan struct{}, par
pm.MaxTimestamp = &maxVal
}
pm.mustWriteMetadata(fileSystem, dstPath)
- fileSystem.SyncPath(dstPath)
+ // No SyncPath: mustWriteMetadata goes through MustFlushAtomic which
+ // already fsyncs the parent directory after rename.
p := mustOpenPart(partID, dstPath, fileSystem)
return newPartWrapper(nil, p), nil
diff --git a/banyand/internal/sidx/merger_durability_test.go b/banyand/internal/sidx/merger_durability_test.go
new file mode 100644
index 000000000..ed29ce6e0
--- /dev/null
+++ b/banyand/internal/sidx/merger_durability_test.go
@@ -0,0 +1,95 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package sidx
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/api/common"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+)
+
+// minimalTestElements builds a tiny elements collection sufficient to flush a
+// non-empty memPart to disk.
+func minimalTestElements(t *testing.T) []testElement {
+ t.Helper()
+ return []testElement{
+ {seriesID: common.SeriesID(1), userKey: 1, data: []byte("a")},
+ {seriesID: common.SeriesID(1), userKey: 2, data: []byte("b")},
+ }
+}
+
+// Test_partFlush_atomicManifest_noTmpLeftover asserts that after a successful
+// part flush, no <file>.tmp sibling survives anywhere under the part directory.
+// Sidx writes manifest.json (not metadata.json), via fs.MustFlushAtomic.
+func Test_partFlush_atomicManifest_noTmpLeftover(t *testing.T) {
+ tempDir := t.TempDir()
+ testFS := fs.NewLocalFileSystem()
+
+ elements := createTestElements(minimalTestElements(t))
+ defer releaseElements(elements)
+
+ mp := GenerateMemPart()
+ defer ReleaseMemPart(mp)
+ mp.mustInitFromElements(elements)
+
+ partDir := filepath.Join(tempDir, "part_1")
+ mp.mustFlush(testFS, partDir)
+
+ require.NoError(t, filepath.Walk(tempDir, func(p string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if info.IsDir() {
+ return nil
+ }
+ assert.Falsef(t, strings.HasSuffix(p, ".tmp"), "leftover tmp at %s", p)
+ return nil
+ }))
+}
+
+// Test_mustOpenPart_removesLeftoverTmp asserts that mustOpenPart removes a
+// stray <file>.tmp sibling whose matching final exists.
+func Test_mustOpenPart_removesLeftoverTmp(t *testing.T) {
+ tempDir := t.TempDir()
+ testFS := fs.NewLocalFileSystem()
+
+ elements := createTestElements(minimalTestElements(t))
+ defer releaseElements(elements)
+
+ mp := GenerateMemPart()
+ mp.mustInitFromElements(elements)
+ partDir := filepath.Join(tempDir, "part_1")
+ mp.mustFlush(testFS, partDir)
+ ReleaseMemPart(mp)
+
+ stray := filepath.Join(partDir, "manifest.json.tmp")
+ require.NoError(t, os.WriteFile(stray, []byte("stale-leftover"), 0o600))
+
+ p := mustOpenPart(1, partDir, testFS)
+ defer p.close()
+
+ _, statErr := os.Stat(stray)
+ assert.Truef(t, os.IsNotExist(statErr), "expected stray .tmp removed, got err=%v", statErr)
+}
diff --git a/banyand/internal/sidx/metadata.go b/banyand/internal/sidx/metadata.go
index 40b2cf521..726a77188 100644
--- a/banyand/internal/sidx/metadata.go
+++ b/banyand/internal/sidx/metadata.go
@@ -455,5 +455,5 @@ func (pm *partMetadata) mustWriteMetadata(fileSystem fs.FileSystem, partPath str
if err != nil {
logger.GetLogger().Panic().Err(err).Str("path", partPath).Msg("failed to marshal part metadata")
}
- fs.MustFlush(fileSystem, manifestData, filepath.Join(partPath, manifestFilename), storage.FilePerm)
+ fs.MustFlushAtomic(fileSystem, manifestData, filepath.Join(partPath, manifestFilename), storage.FilePerm)
}
diff --git a/banyand/internal/sidx/part.go b/banyand/internal/sidx/part.go
index f0372af6a..80a03b4af 100644
--- a/banyand/internal/sidx/part.go
+++ b/banyand/internal/sidx/part.go
@@ -20,6 +20,7 @@ package sidx
import (
"encoding/json"
"fmt"
+ "os"
"path"
"path/filepath"
"sort"
@@ -112,6 +113,12 @@ func mustOpenPart(partID uint64, path string, fileSystem fs.FileSystem) *part {
fileSystem: fileSystem,
}
+ // Remove safe post-rename leftovers (<file>.tmp next to a healthy <file>)
+ // before opening any of the part's files. A .tmp without its final stays
+ // in place so the existing panic-on-missing-metadata fires for operator
+ // intervention.
+ fs.CleanupLeftoverTmp(fileSystem, path)
+
// Open standard files.
p.primary = mustOpenReader(filepath.Join(path, primaryFilename), fileSystem)
p.data = mustOpenReader(filepath.Join(path, dataFilename), fileSystem)
@@ -149,6 +156,15 @@ func (p *part) loadPartMetadata() error {
return nil
}
+ // If a stranded manifest.json.tmp exists without its final, the most
+ // recent atomic commit crashed between fsync(tmp) and rename. Falling
+ // back to meta.bin would silently hide the incomplete-commit state
+ // from the operator. Surface it as an error instead — the canonical
+ // fail-fast contract.
+ if _, statErr := os.Stat(manifestPath + ".tmp"); statErr == nil {
+ return fmt.Errorf("found stranded %s.tmp without %s — incomplete atomic commit, manual recovery required", manifestPath, manifestPath)
+ }
+
// Fallback to meta.bin for backward compatibility
metaData, err := p.fileSystem.Read(filepath.Join(p.path, metaFilename))
if err != nil {
@@ -906,7 +922,8 @@ func (spc *SyncPartContext) Finish() string {
}
if spc.fileSystem != nil && spc.partPath != "" {
spc.partMeta.mustWriteMetadata(spc.fileSystem, spc.partPath)
- spc.fileSystem.SyncPath(spc.partPath)
+ // No SyncPath: mustWriteMetadata goes through MustFlushAtomic which
+ // already fsyncs the parent directory after rename.
}
p := spc.partPath
spc.partPath = ""
diff --git a/banyand/measure/merger.go b/banyand/measure/merger.go
index 206dee25d..9828c10d8 100644
--- a/banyand/measure/merger.go
+++ b/banyand/measure/merger.go
@@ -260,7 +260,8 @@ func (tst *tsTable) mergeParts(fileSystem fs.FileSystem, closeCh <-chan struct{}
return nil, err
}
pm.mustWriteMetadata(fileSystem, dstPath)
- fileSystem.SyncPath(dstPath)
+ // No SyncPath: mustWriteMetadata goes through WriteAtomic which already
+ // fsyncs the parent directory after rename.
p := mustOpenFilePart(partID, root, fileSystem)
return newPartWrapper(nil, p), nil
}
diff --git a/banyand/measure/merger_durability_test.go b/banyand/measure/merger_durability_test.go
new file mode 100644
index 000000000..7ff12d92a
--- /dev/null
+++ b/banyand/measure/merger_durability_test.go
@@ -0,0 +1,82 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+)
+
+// Test_partFlush_atomicMetadata_noTmpLeftover asserts that after a successful
+// part flush, no <file>.tmp sibling survives anywhere under the part directory.
+// This is the post-condition of mustWriteMetadata going through
+// fileSystem.WriteAtomic.
+func Test_partFlush_atomicMetadata_noTmpLeftover(t *testing.T) {
+ tmpPath, defFn := test.Space(require.New(t))
+ defer defFn()
+ fileSystem := fs.NewLocalFileSystem()
+ epoch := uint64(1)
+
+ mp := generateMemPart()
+ defer releaseMemPart(mp)
+ mp.mustInitFromDataPoints(dps)
+ mp.mustFlush(fileSystem, partPath(tmpPath, epoch))
+
+ require.NoError(t, filepath.Walk(tmpPath, func(p string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if info.IsDir() {
+ return nil
+ }
+ assert.Falsef(t, strings.HasSuffix(p, ".tmp"), "leftover tmp at %s", p)
+ return nil
+ }))
+}
+
+// Test_mustOpenFilePart_removesLeftoverTmp asserts that mustOpenFilePart
+// removes a stray <file>.tmp sibling whose matching final exists.
+func Test_mustOpenFilePart_removesLeftoverTmp(t *testing.T) {
+ tmpPath, defFn := test.Space(require.New(t))
+ defer defFn()
+ fileSystem := fs.NewLocalFileSystem()
+ epoch := uint64(1)
+
+ mp := generateMemPart()
+ mp.mustInitFromDataPoints(dps)
+ mp.mustFlush(fileSystem, partPath(tmpPath, epoch))
+ releaseMemPart(mp)
+
+ pp := partPath(tmpPath, epoch)
+ stray := filepath.Join(pp, "metadata.json.tmp")
+ require.NoError(t, os.WriteFile(stray, []byte("stale-leftover"), 0o600))
+
+ p := mustOpenFilePart(epoch, tmpPath, fileSystem)
+ defer p.close()
+
+ _, statErr := os.Stat(stray)
+ assert.Truef(t, os.IsNotExist(statErr), "expected stray .tmp removed, got err=%v", statErr)
+}
diff --git a/banyand/measure/merger_test.go b/banyand/measure/merger_test.go
index 7cdf97b98..a6727781a 100644
--- a/banyand/measure/merger_test.go
+++ b/banyand/measure/merger_test.go
@@ -1457,9 +1457,9 @@ func Test_mergeParts_fileBased(t *testing.T) {
fileSystem := fs.NewLocalFileSystem()
for i := 0; i < numParts; i++ {
startTS := int64(i)*totalPerPart + 1
- dps := generateDatapointsWithMultipleBlocks(startTS, countPerBlock, blocksPerPart)
+ partDPs := generateDatapointsWithMultipleBlocks(startTS, countPerBlock, blocksPerPart)
mp := generateMemPart()
- mp.mustInitFromDataPoints(dps)
+ mp.mustInitFromDataPoints(partDPs)
mp.mustFlush(fileSystem, partPath(tmpPath, uint64(i+1)))
filePW := newPartWrapper(nil, mustOpenFilePart(uint64(i+1), tmpPath, fileSystem))
filePW.p.partMetadata.ID = uint64(i + 1)
diff --git a/banyand/measure/part.go b/banyand/measure/part.go
index 844713fb1..93f5b33c5 100644
--- a/banyand/measure/part.go
+++ b/banyand/measure/part.go
@@ -218,14 +218,18 @@ func (mp *memPart) mustFlush(fileSystem fs.FileSystem, path string) {
fs.MustFlush(fileSystem, tfh.Buf, filepath.Join(path, name+tagFamiliesMetadataFilenameExt), storage.FilePerm)
}
- // Flush series metadata if available
+ // Flush series metadata if available. Goes through MustFlushAtomic
+ // because the file is read at part open by mustOpenFilePart and a
+ // crash during a non-atomic write could leave a partial file that
+ // reads as corrupt.
if len(mp.seriesMetadata.Buf) > 0 {
- fs.MustFlush(fileSystem, mp.seriesMetadata.Buf, filepath.Join(path, seriesMetadataFilename), storage.FilePerm)
+ fs.MustFlushAtomic(fileSystem, mp.seriesMetadata.Buf, filepath.Join(path, seriesMetadataFilename), storage.FilePerm)
}
mp.partMetadata.mustWriteMetadata(fileSystem, path)
-
- fileSystem.SyncPath(path)
+ // No SyncPath: mustWriteMetadata goes through WriteAtomic which already
+ // fsyncs the parent directory after rename, covering the dirent changes
+ // for all data files written above.
}
func uncompressedDataPointSizeBytes(index int, dps *dataPoints) uint64 {
@@ -310,6 +314,7 @@ func mustOpenFilePart(id uint64, root string, fileSystem fs.FileSystem) *part {
partPath := partPath(root, id)
p.path = partPath
p.fileSystem = fileSystem
+ fs.CleanupLeftoverTmp(fileSystem, partPath)
p.partMetadata.mustReadMetadata(fileSystem, partPath)
p.partMetadata.ID = id
diff --git a/banyand/measure/part_metadata.go b/banyand/measure/part_metadata.go
index dbd41b5f6..e7458c07d 100644
--- a/banyand/measure/part_metadata.go
+++ b/banyand/measure/part_metadata.go
@@ -97,7 +97,7 @@ func (pm *partMetadata) mustWriteMetadata(fileSystem fs.FileSystem, partPath str
return
}
metadataPath := filepath.Join(partPath, metadataFilename)
- n, err := fileSystem.Write(metadata, metadataPath, storage.FilePerm)
+ n, err := fileSystem.WriteAtomic(metadata, metadataPath, storage.FilePerm)
if err != nil {
logger.Panicf("cannot write metadata: %s", err)
return
diff --git a/banyand/measure/tstable.go b/banyand/measure/tstable.go
index 34e989601..4e3210d20 100644
--- a/banyand/measure/tstable.go
+++ b/banyand/measure/tstable.go
@@ -259,27 +259,12 @@ func (tst *tsTable) mustWriteSnapshot(snapshot uint64, partNames []string) {
logger.Panicf("cannot marshal partNames to JSON: %s", err)
}
snapshotPath := filepath.Join(tst.root, snapshotName(snapshot))
- snapshotTempPath := snapshotPath + ".tmp"
- lf, err := tst.fileSystem.CreateLockFile(snapshotTempPath, storage.FilePerm)
- if err != nil {
- logger.Panicf("cannot create lock file %s: %s", snapshotTempPath, err)
- }
- n, err := lf.Write(data)
- if err != nil {
- _ = lf.Close()
- logger.Panicf("cannot write snapshot %s: %s", snapshotTempPath, err)
- }
- if n != len(data) {
- _ = lf.Close()
- logger.Panicf("unexpected number of bytes written to %s; got %d; want %d", snapshotTempPath, n, len(data))
- }
- if closeErr := lf.Close(); closeErr != nil {
- logger.Panicf("cannot close snapshot temp file %s: %s", snapshotTempPath, closeErr)
- }
- if renameErr := tst.fileSystem.Rename(snapshotTempPath, snapshotPath); renameErr != nil {
- logger.Panicf("cannot rename snapshot %s to %s: %s", snapshotTempPath, snapshotPath, renameErr)
- }
- tst.fileSystem.SyncPath(tst.root)
+ // WriteAtomic does write-tmp -> file.Sync -> rename -> dir.Sync, so a
+ // crash leaves either the previous valid snapshot or only <name>.tmp,
+ // never a renamed-but-not-durable snapshot. The orphan-cleanup logic
+ // at startup depends on snapshots being authoritative; this closes the
+ // last metadata-style write site that could undermine that assumption.
+ fs.MustFlushAtomic(tst.fileSystem, data, snapshotPath, storage.FilePerm)
}
func (tst *tsTable) readSnapshot(snapshot uint64) ([]uint64, error) {
diff --git a/banyand/measure/write_data.go b/banyand/measure/write_data.go
index 2ecf535bf..c37b52e60 100644
--- a/banyand/measure/write_data.go
+++ b/banyand/measure/write_data.go
@@ -57,7 +57,8 @@ func (s *syncPartContext) FinishSync() error {
s.releaseCoreWriters()
s.partMeta.mustWriteMetadata(s.fileSystem, s.partPath)
- s.fileSystem.SyncPath(s.partPath)
+ // No SyncPath: mustWriteMetadata goes through WriteAtomic which already
+ // fsyncs the parent directory after rename.
s.tsTable.mustAddFilePart(s.partID)
s.partPath = ""
diff --git a/banyand/stream/merger.go b/banyand/stream/merger.go
index ba7486878..934eb43cb 100644
--- a/banyand/stream/merger.go
+++ b/banyand/stream/merger.go
@@ -259,7 +259,8 @@ func (tst *tsTable) mergeParts(fileSystem fs.FileSystem, closeCh <-chan struct{}
return nil, err
}
pm.mustWriteMetadata(fileSystem, dstPath)
- fileSystem.SyncPath(dstPath)
+ // No SyncPath: mustWriteMetadata goes through WriteAtomic which already
+ // fsyncs the parent directory after rename.
p := mustOpenFilePart(partID, root, fileSystem)
return newPartWrapper(nil, p), nil
diff --git a/banyand/stream/merger_durability_test.go b/banyand/stream/merger_durability_test.go
new file mode 100644
index 000000000..e86d67f0a
--- /dev/null
+++ b/banyand/stream/merger_durability_test.go
@@ -0,0 +1,80 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package stream
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+)
+
+// Test_partFlush_atomicMetadata_noTmpLeftover asserts that after a successful
+// part flush, no <file>.tmp sibling survives anywhere under the part directory.
+func Test_partFlush_atomicMetadata_noTmpLeftover(t *testing.T) {
+ tmpPath, defFn := test.Space(require.New(t))
+ defer defFn()
+ fileSystem := fs.NewLocalFileSystem()
+ epoch := uint64(1)
+
+ mp := generateMemPart()
+ defer releaseMemPart(mp)
+ mp.mustInitFromElements(es)
+ mp.mustFlush(fileSystem, partPath(tmpPath, epoch))
+
+ require.NoError(t, filepath.Walk(tmpPath, func(p string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if info.IsDir() {
+ return nil
+ }
+ assert.Falsef(t, strings.HasSuffix(p, ".tmp"), "leftover tmp at %s", p)
+ return nil
+ }))
+}
+
+// Test_mustOpenFilePart_removesLeftoverTmp asserts that mustOpenFilePart
+// removes a stray <file>.tmp sibling whose matching final exists.
+func Test_mustOpenFilePart_removesLeftoverTmp(t *testing.T) {
+ tmpPath, defFn := test.Space(require.New(t))
+ defer defFn()
+ fileSystem := fs.NewLocalFileSystem()
+ epoch := uint64(1)
+
+ mp := generateMemPart()
+ mp.mustInitFromElements(es)
+ mp.mustFlush(fileSystem, partPath(tmpPath, epoch))
+ releaseMemPart(mp)
+
+ pp := partPath(tmpPath, epoch)
+ stray := filepath.Join(pp, "metadata.json.tmp")
+ require.NoError(t, os.WriteFile(stray, []byte("stale-leftover"), 0o600))
+
+ p := mustOpenFilePart(epoch, tmpPath, fileSystem)
+ defer p.close()
+
+ _, statErr := os.Stat(stray)
+ assert.Truef(t, os.IsNotExist(statErr), "expected stray .tmp removed, got err=%v", statErr)
+}
diff --git a/banyand/stream/part.go b/banyand/stream/part.go
index b727f5d7a..7ee11504c 100644
--- a/banyand/stream/part.go
+++ b/banyand/stream/part.go
@@ -223,14 +223,18 @@ func (mp *memPart) mustFlush(fileSystem fs.FileSystem, path string) {
fs.MustFlush(fileSystem, tfh.Buf, filepath.Join(path, name+tagFamiliesFilterFilenameExt), storage.FilePerm)
}
- // Flush series metadata if available
+ // Flush series metadata if available. Goes through MustFlushAtomic
+ // because the file is read at part open by mustOpenFilePart and a
+ // crash during a non-atomic write could leave a partial file that
+ // reads as corrupt.
if len(mp.seriesMetadata.Buf) > 0 {
- fs.MustFlush(fileSystem, mp.seriesMetadata.Buf, filepath.Join(path, seriesMetadataFilename), storage.FilePerm)
+ fs.MustFlushAtomic(fileSystem, mp.seriesMetadata.Buf, filepath.Join(path, seriesMetadataFilename), storage.FilePerm)
}
mp.partMetadata.mustWriteMetadata(fileSystem, path)
-
- fileSystem.SyncPath(path)
+ // No SyncPath: mustWriteMetadata goes through WriteAtomic which already
+ // fsyncs the parent directory after rename, covering the dirent changes
+ // for all data files written above.
}
func uncompressedElementSizeBytes(index int, es *elements) uint64 {
@@ -311,6 +315,7 @@ func mustOpenFilePart(id uint64, root string, fileSystem fs.FileSystem) *part {
partPath := partPath(root, id)
p.path = partPath
p.fileSystem = fileSystem
+ fs.CleanupLeftoverTmp(fileSystem, partPath)
p.partMetadata.mustReadMetadata(fileSystem, partPath)
p.partMetadata.ID = id
diff --git a/banyand/stream/part_metadata.go b/banyand/stream/part_metadata.go
index 25c0eaa2f..47cddb501 100644
--- a/banyand/stream/part_metadata.go
+++ b/banyand/stream/part_metadata.go
@@ -97,7 +97,7 @@ func (pm *partMetadata) mustWriteMetadata(fileSystem fs.FileSystem, partPath str
return
}
metadataPath := filepath.Join(partPath, metadataFilename)
- n, err := fileSystem.Write(metadata, metadataPath, storage.FilePerm)
+ n, err := fileSystem.WriteAtomic(metadata, metadataPath, storage.FilePerm)
if err != nil {
logger.Panicf("cannot write metadata: %s", err)
return
diff --git a/banyand/stream/tstable.go b/banyand/stream/tstable.go
index 40f72581f..aa062a7d4 100644
--- a/banyand/stream/tstable.go
+++ b/banyand/stream/tstable.go
@@ -156,27 +156,12 @@ func (tst *tsTable) mustWriteSnapshot(snapshot uint64, partNames []string) {
logger.Panicf("cannot marshal partNames to JSON: %s", err)
}
snapshotPath := filepath.Join(tst.root, snapshotName(snapshot))
- snapshotTempPath := snapshotPath + ".tmp"
- lf, err := tst.fileSystem.CreateLockFile(snapshotTempPath, storage.FilePerm)
- if err != nil {
- logger.Panicf("cannot create lock file %s: %s", snapshotTempPath, err)
- }
- n, err := lf.Write(data)
- if err != nil {
- _ = lf.Close()
- logger.Panicf("cannot write snapshot %s: %s", snapshotTempPath, err)
- }
- if n != len(data) {
- _ = lf.Close()
- logger.Panicf("unexpected number of bytes written to %s; got %d; want %d", snapshotTempPath, n, len(data))
- }
- if closeErr := lf.Close(); closeErr != nil {
- logger.Panicf("cannot close snapshot temp file %s: %s", snapshotTempPath, closeErr)
- }
- if renameErr := tst.fileSystem.Rename(snapshotTempPath, snapshotPath); renameErr != nil {
- logger.Panicf("cannot rename snapshot %s to %s: %s", snapshotTempPath, snapshotPath, renameErr)
- }
- tst.fileSystem.SyncPath(tst.root)
+ // WriteAtomic does write-tmp -> file.Sync -> rename -> dir.Sync, so a
+ // crash leaves either the previous valid snapshot or only <name>.tmp,
+ // never a renamed-but-not-durable snapshot. The orphan-cleanup logic
+ // at startup depends on snapshots being authoritative; this closes the
+ // last metadata-style write site that could undermine that assumption.
+ fs.MustFlushAtomic(tst.fileSystem, data, snapshotPath, storage.FilePerm)
}
func (tst *tsTable) readSnapshot(snapshot uint64) ([]uint64, error) {
diff --git a/banyand/stream/write_data.go b/banyand/stream/write_data.go
index 6a6ff24da..76f4e4ced 100644
--- a/banyand/stream/write_data.go
+++ b/banyand/stream/write_data.go
@@ -56,7 +56,8 @@ func (s *syncPartContext) FinishSync() error {
s.releaseCoreWriters()
s.partMeta.mustWriteMetadata(s.fileSystem, s.partPath)
- s.fileSystem.SyncPath(s.partPath)
+ // No SyncPath: mustWriteMetadata goes through WriteAtomic which already
+ // fsyncs the parent directory after rename.
s.tsTable.mustAddFilePart(s.partID)
s.partPath = ""
diff --git a/banyand/trace/merger.go b/banyand/trace/merger.go
index 480897746..d4286fd9d 100644
--- a/banyand/trace/merger.go
+++ b/banyand/trace/merger.go
@@ -352,7 +352,9 @@ func (tst *tsTable) mergeParts(fileSystem fs.FileSystem, closeCh <-chan struct{}
tf.mustWriteTraceIDFilter(fileSystem, dstPath)
tf.reset()
tt.mustWriteTagType(fileSystem, dstPath)
- fileSystem.SyncPath(dstPath)
+ // No SyncPath here: each mustWrite* helper goes through fileSystem.WriteAtomic
+ // which already fsyncs the parent directory after rename. The last atomic
+ // metadata write covers all prior dirent changes (data file creations).
p := mustOpenFilePart(partID, root, fileSystem)
return newPartWrapper(nil, p), nil
}
diff --git a/banyand/trace/merger_durability_test.go b/banyand/trace/merger_durability_test.go
new file mode 100644
index 000000000..6dfb9dcb2
--- /dev/null
+++ b/banyand/trace/merger_durability_test.go
@@ -0,0 +1,179 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package trace
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/banyand/protector"
+ "github.com/apache/skywalking-banyandb/pkg/fs"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+)
+
+// Test_partFlush_atomicMetadata_noTmpLeftover asserts that after a successful
+// part flush, no <file>.tmp sibling survives anywhere under the part directory.
+// This is the post-condition of mustWriteMetadata, mustWriteTraceIDFilter, and
+// mustWriteTagType all going through fileSystem.WriteAtomic.
+func Test_partFlush_atomicMetadata_noTmpLeftover(t *testing.T) {
+ tmpPath, defFn := test.Space(require.New(t))
+ defer defFn()
+ fileSystem := fs.NewLocalFileSystem()
+ epoch := uint64(1)
+
+ mp := generateMemPart()
+ defer releaseMemPart(mp)
+ mp.mustInitFromTraces(ts)
+ mp.mustFlush(fileSystem, partPath(tmpPath, epoch))
+
+ require.NoError(t, filepath.Walk(tmpPath, func(p string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if info.IsDir() {
+ return nil
+ }
+ assert.Falsef(t, strings.HasSuffix(p, ".tmp"), "leftover tmp at %s", p)
+ return nil
+ }))
+}
+
+// Test_mustOpenFilePart_removesLeftoverTmp asserts that mustOpenFilePart
+// removes a stray <file>.tmp sibling whose matching final exists. This is the
+// post-rename cleanup hook into fs.CleanupLeftoverTmp. Trace writes three
+// metadata files via WriteAtomic — metadata.json, traceID.filter, tag.type —
+// so all three .tmp shapes must be cleaned up on open.
+func Test_mustOpenFilePart_removesLeftoverTmp(t *testing.T) {
+ cases := []string{"metadata.json.tmp", "traceID.filter.tmp", "tag.type.tmp"}
+ for _, strayName := range cases {
+ t.Run(strayName, func(t *testing.T) {
+ tmpPath, defFn := test.Space(require.New(t))
+ defer defFn()
+ fileSystem := fs.NewLocalFileSystem()
+ epoch := uint64(1)
+
+ mp := generateMemPart()
+ mp.mustInitFromTraces(ts)
+ mp.mustFlush(fileSystem, partPath(tmpPath, epoch))
+ releaseMemPart(mp)
+
+ pp := partPath(tmpPath, epoch)
+ // Skip the case where the matching final file doesn't exist on
+ // disk for this fixture — without a healthy <file>, the cleanup
+ // helper is correct to leave the .tmp in place.
+ final := strings.TrimSuffix(strayName, ".tmp")
+ if _, err := os.Stat(filepath.Join(pp, final)); err != nil {
+ t.Skipf("fixture did not produce %s; cleanup behavior covered by other cases", final)
+ }
+ stray := filepath.Join(pp, strayName)
+ require.NoError(t, os.WriteFile(stray, []byte("stale-leftover"), 0o600))
+
+ p := mustOpenFilePart(epoch, tmpPath, fileSystem)
+ defer p.close()
+
+ _, statErr := os.Stat(stray)
+ assert.Truef(t, os.IsNotExist(statErr), "expected %s removed, got err=%v", strayName, statErr)
+ })
+ }
+}
+
+// Test_merger_tornSpansBin_stillPanics is the reproducer described in the body
+// of apache/skywalking#13862. It forges the production-shape on-disk state —
+// a part directory whose metadata.json claims N blocks but whose spans.bin is
+// truncated — and feeds it to mergeParts. The expected outcome under the
+// merge-durability fix is *unchanged* from before the fix: the merger panics
+// with "offset must be equal to bytesRead", because that is BanyanDB's
+// canonical fail-fast contract for a corrupt source part.
+//
+// The fix prevents torn parts from being CREATED via a clean-crash path
+// (atomic metadata commit makes that impossible). It does NOT prevent the
+// panic when an externally-corrupt part is forced onto disk (rsync, manual
+// file copy, host disk failure, pre-fix-deployment leftovers). This test
+// guards against accidental panic suppression by future refactors.
+func Test_merger_tornSpansBin_stillPanics(t *testing.T) {
+ tmpPath, defFn := test.Space(require.New(t))
+ defer defFn()
+ fileSystem := fs.NewLocalFileSystem()
+
+ // Build two healthy parts with multiple blocks.
+ mp1 := generateMemPart()
+ mp1.mustInitFromTraces(ts)
+ mp1.mustFlush(fileSystem, partPath(tmpPath, 1))
+ releaseMemPart(mp1)
+
+ mp2 := generateMemPart()
+ mp2.mustInitFromTraces(ts)
+ mp2.mustFlush(fileSystem, partPath(tmpPath, 2))
+ releaseMemPart(mp2)
+
+ // Forge the torn shape: truncate part 1's spans.bin so primary.bin's
+ // block-metadata claims more bytes than spans.bin actually contains.
+ spansPath := filepath.Join(partPath(tmpPath, 1), "spans.bin")
+ stat, err := os.Stat(spansPath)
+ require.NoError(t, err)
+ require.Greater(t, stat.Size(), int64(0), "test fixture must produce non-empty spans.bin")
+ tornAt := stat.Size() - 1
+ require.NoError(t, os.Truncate(spansPath, tornAt))
+
+ // mustOpenFilePart succeeds because metadata.json is intact — the
+ // truncation is only in spans.bin.
+ p1 := mustOpenFilePart(1, tmpPath, fileSystem)
+ p1.partMetadata.ID = 1
+ p2 := mustOpenFilePart(2, tmpPath, fileSystem)
+ p2.partMetadata.ID = 2
+
+ tst := &tsTable{pm: protector.Nop{}, fileSystem: fileSystem, root: tmpPath}
+ closeCh := make(chan struct{})
+ defer close(closeCh)
+
+ defer func() {
+ r := recover()
+ // Release fds even on the panic path. mustOpenFilePart opens
+ // many underlying files; without explicit close here the panic
+ // would leak them for the lifetime of the test process.
+ p1.close()
+ p2.close()
+ require.NotNilf(t, r, "merger must panic on torn spans.bin (canonical fail-fast contract)")
+ // Accept any panic from the merger's read path. The exact message
+ // depends on which block boundary the truncation hit; valid shapes:
+ // "offset N must be equal to bytesRead M"
+ // "cannot read full data: ..."
+ // "cannot read data: ..."
+ // "cannot decode spans: ..."
+ got := fmt.Sprint(r)
+ t.Logf("captured panic: %s", got)
+ valid := strings.Contains(got, "offset") ||
+ strings.Contains(got, "cannot read") ||
+ strings.Contains(got, "cannot decode") ||
+ strings.Contains(got, "EOF")
+ assert.Truef(t, valid, "panic message did not match canonical fail-fast shapes: %s", got)
+ }()
+
+ _, _ = tst.mergeParts(fileSystem, closeCh, []*partWrapper{
+ newPartWrapper(nil, p1),
+ newPartWrapper(nil, p2),
+ }, 99, tmpPath)
+
+ t.Fatal("mergeParts returned without panicking — torn-part fail-fast contract is broken")
+}
diff --git a/banyand/trace/part.go b/banyand/trace/part.go
index 3ed531967..353700cf0 100644
--- a/banyand/trace/part.go
+++ b/banyand/trace/part.go
@@ -296,16 +296,20 @@ func (mp *memPart) mustFlush(fileSystem fs.FileSystem, path string) {
fs.MustFlush(fileSystem, tm.Buf, filepath.Join(path, name+tagsMetadataFilenameExt), storage.FilePerm)
}
- // Flush series metadata if available
+ // Flush series metadata if available. Goes through MustFlushAtomic
+ // because the file is read at part open by mustOpenFilePart and a
+ // crash during a non-atomic write could leave a partial file that
+ // reads as corrupt.
if len(mp.seriesMetadata.Buf) > 0 {
- fs.MustFlush(fileSystem, mp.seriesMetadata.Buf, filepath.Join(path, seriesMetadataFilename), storage.FilePerm)
+ fs.MustFlushAtomic(fileSystem, mp.seriesMetadata.Buf, filepath.Join(path, seriesMetadataFilename), storage.FilePerm)
}
mp.partMetadata.mustWriteMetadata(fileSystem, path)
mp.tagType.mustWriteTagType(fileSystem, path)
mp.traceIDFilter.mustWriteTraceIDFilter(fileSystem, path)
-
- fileSystem.SyncPath(path)
+ // No SyncPath: each mustWrite* helper uses WriteAtomic which fsyncs the
+ // parent directory after rename, covering all prior dirent changes
+ // (data file creations).
}
func generateMemPart() *memPart {
@@ -373,6 +377,7 @@ func mustOpenFilePart(id uint64, root string, fileSystem fs.FileSystem) *part {
partPath := partPath(root, id)
p.path = partPath
p.fileSystem = fileSystem
+ fs.CleanupLeftoverTmp(fileSystem, partPath)
p.partMetadata.mustReadMetadata(fileSystem, partPath)
p.partMetadata.ID = id
diff --git a/banyand/trace/part_metadata.go b/banyand/trace/part_metadata.go
index ae1a029a2..62468b444 100644
--- a/banyand/trace/part_metadata.go
+++ b/banyand/trace/part_metadata.go
@@ -102,7 +102,7 @@ func (pm *partMetadata) mustWriteMetadata(fileSystem fs.FileSystem, partPath str
return
}
metadataPath := filepath.Join(partPath, metadataFilename)
- n, err := fileSystem.Write(metadata, metadataPath, storage.FilePerm)
+ n, err := fileSystem.WriteAtomic(metadata, metadataPath, storage.FilePerm)
if err != nil {
logger.Panicf("cannot write metadata: %s", err)
return
@@ -199,7 +199,7 @@ func (tt tagType) mustWriteTagType(fileSystem fs.FileSystem, partPath string) {
data = tt.marshal(data)
tagTypePath := filepath.Join(partPath, tagTypeFilename)
- n, err := fileSystem.Write(data, tagTypePath, storage.FilePerm)
+ n, err := fileSystem.WriteAtomic(data, tagTypePath, storage.FilePerm)
if err != nil {
logger.Panicf("cannot write tagType: %s", err)
return
@@ -249,7 +249,7 @@ func (tf *traceIDFilter) mustWriteTraceIDFilter(fileSystem fs.FileSystem, partPa
data = encodeBloomFilter(data, tf.filter)
traceIDFilterPath := filepath.Join(partPath, traceIDFilterFilename)
- n, err := fileSystem.Write(data, traceIDFilterPath, storage.FilePerm)
+ n, err := fileSystem.WriteAtomic(data, traceIDFilterPath, storage.FilePerm)
if err != nil {
logger.Panicf("cannot write traceIDFilter: %s", err)
return
diff --git a/banyand/trace/tstable.go b/banyand/trace/tstable.go
index cad0dea0b..dd13bc9b9 100644
--- a/banyand/trace/tstable.go
+++ b/banyand/trace/tstable.go
@@ -157,27 +157,12 @@ func (tst *tsTable) mustWriteSnapshot(snapshot uint64, partNames []string) {
logger.Panicf("cannot marshal partNames to JSON: %s", err)
}
snapshotPath := filepath.Join(tst.root, snapshotName(snapshot))
- snapshotTempPath := snapshotPath + ".tmp"
- lf, err := tst.fileSystem.CreateLockFile(snapshotTempPath, storage.FilePerm)
- if err != nil {
- logger.Panicf("cannot create lock file %s: %s", snapshotTempPath, err)
- }
- n, err := lf.Write(data)
- if err != nil {
- _ = lf.Close()
- logger.Panicf("cannot write snapshot %s: %s", snapshotTempPath, err)
- }
- if n != len(data) {
- _ = lf.Close()
- logger.Panicf("unexpected number of bytes written to %s; got %d; want %d", snapshotTempPath, n, len(data))
- }
- if closeErr := lf.Close(); closeErr != nil {
- logger.Panicf("cannot close snapshot temp file %s: %s", snapshotTempPath, closeErr)
- }
- if renameErr := tst.fileSystem.Rename(snapshotTempPath, snapshotPath); renameErr != nil {
- logger.Panicf("cannot rename snapshot %s to %s: %s", snapshotTempPath, snapshotPath, renameErr)
- }
- tst.fileSystem.SyncPath(tst.root)
+ // WriteAtomic does write-tmp -> file.Sync -> rename -> dir.Sync, so a
+ // crash leaves either the previous valid snapshot or only <name>.tmp,
+ // never a renamed-but-not-durable snapshot. The orphan-cleanup logic
+ // at startup depends on snapshots being authoritative; this closes the
+ // last metadata-style write site that could undermine that assumption.
+ fs.MustFlushAtomic(tst.fileSystem, data, snapshotPath, storage.FilePerm)
}
func (tst *tsTable) readSnapshot(snapshot uint64) ([]uint64, error) {
diff --git a/banyand/trace/write_data.go b/banyand/trace/write_data.go
index 0d339d71f..16c989b52 100644
--- a/banyand/trace/write_data.go
+++ b/banyand/trace/write_data.go
@@ -102,13 +102,14 @@ func (s *syncPartContext) FinishSync() error {
s.releaseCoreWriters()
if len(s.traceIDFilterBuffer) > 0 {
- fs.MustFlush(s.fileSystem, s.traceIDFilterBuffer, filepath.Join(s.partPath, traceIDFilterFilename), storage.FilePerm)
+ fs.MustFlushAtomic(s.fileSystem, s.traceIDFilterBuffer, filepath.Join(s.partPath, traceIDFilterFilename), storage.FilePerm)
}
if len(s.tagTypeBuffer) > 0 {
- fs.MustFlush(s.fileSystem, s.tagTypeBuffer, filepath.Join(s.partPath, tagTypeFilename), storage.FilePerm)
+ fs.MustFlushAtomic(s.fileSystem, s.tagTypeBuffer, filepath.Join(s.partPath, tagTypeFilename), storage.FilePerm)
}
s.partMeta.mustWriteMetadata(s.fileSystem, s.partPath)
- s.fileSystem.SyncPath(s.partPath)
+ // No SyncPath: mustWriteMetadata goes through WriteAtomic which already
+ // fsyncs the parent directory after rename.
// Finish SIDX writers and collect file paths for file-backed parts.
sidxFilePartsMap := make(map[string]string, len(s.sidxPartContexts))
diff --git a/pkg/fs/file_system.go b/pkg/fs/file_system.go
index 2254fb29b..3cdc57759 100644
--- a/pkg/fs/file_system.go
+++ b/pkg/fs/file_system.go
@@ -20,6 +20,9 @@ package fs
import (
"io"
+ "os"
+ "path/filepath"
+ "strings"
"github.com/apache/skywalking-banyandb/pkg/logger"
)
@@ -110,6 +113,17 @@ type FileSystem interface {
OpenFile(name string) (File, error)
// Flush mode, which flushes all data to one file.
Write(buffer []byte, name string, permission Mode) (int, error)
+ // WriteAtomic writes buffer to a fresh ".tmp" sibling, fsyncs it, renames it
+ // over name, and fsyncs the parent directory. After a successful return the
+ // content is durable on a POSIX-compliant local filesystem (ext4, xfs, apfs,
+ // ntfs). On any error the .tmp file may be left behind for next-boot cleanup
+ // (when the matching final file exists) or operator intervention (when only
+ // the .tmp exists, indicating a crash before rename).
+ //
+ // Atomicity caveat: os.Rename is not guaranteed atomic on network-mounted
+ // filesystems (NFS, SMB, FUSE-based S3 gateways). BanyanDB targets local
+ // disks; do not run on a network FS without verifying the mount semantics.
+ WriteAtomic(buffer []byte, name string, permission Mode) (int, error)
// Read the entire file using streaming read.
Read(name string) ([]byte, error)
// Delete the file.
@@ -166,6 +180,51 @@ func MustFlush(fs FileSystem, buffer []byte, name string, permission Mode) {
}
}
+// MustFlushAtomic writes buffer to name via WriteAtomic and panics on error.
+// Same panic-on-error semantics as MustFlush; use this when the caller needs
+// crash-safety on the file (metadata.json, manifest.json, etc.).
+func MustFlushAtomic(fs FileSystem, buffer []byte, name string, permission Mode) {
+ n, err := fs.WriteAtomic(buffer, name, permission)
+ if err != nil {
+ logger.GetLogger().Panic().Err(err).Str("path", name).Msg("cannot write data atomically")
+ }
+ if n != len(buffer) {
+ logger.GetLogger().Panic().Int("written", n).Int("expected", len(buffer)).Str("path", name).Msg("BUG: WriteAtomic wrote wrong number of bytes")
+ }
+}
+
+// CleanupLeftoverTmp removes any "<file>.tmp" sibling under partPath whose
+// matching final "<file>" exists. A .tmp without its final indicates a crash
+// between WriteAtomic's fsync(tmp) and rename — that case is left alone so
+// the engine's existing panic-on-missing-metadata fires for operator
+// intervention, per BanyanDB's canonical fail-fast contract.
+//
+// Symlinks ending in ".tmp" are skipped: BanyanDB part directories never
+// contain symlinks, so encountering one is unexpected and we do not want to
+// follow it during cleanup.
+func CleanupLeftoverTmp(fs FileSystem, partPath string) {
+ for _, e := range fs.ReadDir(partPath) {
+ if e.IsDir() {
+ continue
+ }
+ name := e.Name()
+ if !strings.HasSuffix(name, ".tmp") {
+ continue
+ }
+ tmpFull := filepath.Join(partPath, name)
+ info, lstatErr := os.Lstat(tmpFull)
+ if lstatErr != nil || info.Mode()&os.ModeSymlink != 0 {
+ continue
+ }
+ final := name[:len(name)-len(".tmp")]
+ if _, err := os.Stat(filepath.Join(partPath, final)); err == nil {
+ if removeErr := os.Remove(tmpFull); removeErr != nil {
+ logger.GetLogger().Debug().Err(removeErr).Str("path", tmpFull).Msg("CleanupLeftoverTmp: cannot remove leftover .tmp")
+ }
+ }
+ }
+}
+
// MustWriteData writes data to w and panics if it cannot write all data.
func MustWriteData(w SeqWriter, data []byte) {
if len(data) == 0 {
diff --git a/pkg/fs/local_file_system.go b/pkg/fs/local_file_system.go
index aa654050c..4be28b6c5 100644
--- a/pkg/fs/local_file_system.go
+++ b/pkg/fs/local_file_system.go
@@ -228,7 +228,88 @@ func (fs *localFileSystem) Write(buffer []byte, name string, permission Mode) (i
Message: fmt.Sprintf("Flush file return error, file name: %s,error message: %s", name, err),
}
}
+ if syncErr := file.Sync(); syncErr != nil {
+ return size, &FileSystemError{
+ Code: flushError,
+ Message: fmt.Sprintf("Sync file return error, file name: %s, error message: %s", name, syncErr),
+ }
+ }
+
+ return size, nil
+}
+
+// invokeTestHookAfterTmpFsync is a no-op in production. The test build
+// overrides it via pkg/fs/local_file_system_testhook_test.go to simulate a
+// crash between WriteAtomic's fsync(tmp) and rename.
+var invokeTestHookAfterTmpFsync = func() {}
+// WriteAtomic writes buffer to name+".tmp", fsyncs it, renames over name, and
+// fsyncs the parent directory. See FileSystem.WriteAtomic for semantics.
+//
+// On a re-tried call after a prior crash mid-rename, a stale ".tmp" sibling
+// from the prior attempt is overwritten via O_TRUNC. The "next-boot
+// inspection" semantics in the FileSystem interface docstring still hold:
+// CleanupLeftoverTmp on the next part open will remove the .tmp once a
+// matching final exists, and a .tmp without its final remains until the
+// next successful WriteAtomic to the same name.
+func (fs *localFileSystem) WriteAtomic(buffer []byte, name string, permission Mode) (int, error) {
+ tmpName := name + ".tmp"
+ parentDir := filepath.Dir(name)
+ file, err := os.OpenFile(tmpName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.FileMode(permission))
+ if err != nil {
+ if os.IsPermission(err) {
+ return 0, &FileSystemError{
+ Code: permissionError,
+ Message: fmt.Sprintf("There is not enough permission, file name: %s, permission: %d, error message: %s", tmpName, permission, err),
+ }
+ }
+ return 0, &FileSystemError{
+ Code: otherError,
+ Message: fmt.Sprintf("Create tmp file return error, file name: %s, error message: %s", tmpName, err),
+ }
+ }
+ size, writeErr := file.Write(buffer)
+ if writeErr != nil {
+ _ = file.Close()
+ _ = os.Remove(tmpName)
+ return size, &FileSystemError{
+ Code: flushError,
+ Message: fmt.Sprintf("Write tmp file return error, file name: %s, error message: %s", tmpName, writeErr),
+ }
+ }
+ if syncErr := file.Sync(); syncErr != nil {
+ _ = file.Close()
+ _ = os.Remove(tmpName)
+ return size, &FileSystemError{
+ Code: flushError,
+ Message: fmt.Sprintf("Sync tmp file return error, file name: %s, error message: %s", tmpName, syncErr),
+ }
+ }
+ if closeErr := file.Close(); closeErr != nil {
+ _ = os.Remove(tmpName)
+ return size, &FileSystemError{
+ Code: closeError,
+ Message: fmt.Sprintf("Close tmp file return error, file name: %s, error message: %s", tmpName, closeErr),
+ }
+ }
+ invokeTestHookAfterTmpFsync()
+ if renameErr := os.Rename(tmpName, name); renameErr != nil {
+ // Do NOT remove the .tmp on rename failure: it is fully written
+ // and fsynced — the only durable record of what we tried to
+ // commit — and an operator may need it for forensic recovery.
+ // The next successful WriteAtomic call to the same name will
+ // overwrite it via O_TRUNC, so no orphan accumulates.
+ return size, &FileSystemError{
+ Code: otherError,
+ Message: fmt.Sprintf("Rename %s -> %s return error: %s", tmpName, name, renameErr),
+ }
+ }
+ if err := syncDir(parentDir); err != nil {
+ return size, &FileSystemError{
+ Code: flushError,
+ Message: fmt.Sprintf("Sync parent dir return error, dir name: %s, error message: %s", parentDir, err),
+ }
+ }
return size, nil
}
@@ -661,9 +742,21 @@ func (w *seqWriter) Close() error {
Message: fmt.Sprintf("Flush File error, directory name: %s, error message: %s", w.fileName, err),
}
}
-
- if w.file != nil && !w.skipFadvise {
- _ = SyncAndDropCache(w.file.Fd(), 0, 0)
+ if w.file == nil {
+ return nil
+ }
+ if !w.skipFadvise {
+ if syncErr := SyncAndDropCache(w.file.Fd(), 0, 0); syncErr != nil {
+ return &FileSystemError{
+ Code: flushError,
+ Message: fmt.Sprintf("Sync File error, directory name: %s, error message: %s", w.fileName, syncErr),
+ }
+ }
+ } else if syncErr := w.file.Sync(); syncErr != nil {
+ return &FileSystemError{
+ Code: flushError,
+ Message: fmt.Sprintf("Sync File error, directory name: %s, error message: %s", w.fileName, syncErr),
+ }
}
return nil
}
diff --git a/pkg/fs/local_file_system_darwin.go b/pkg/fs/local_file_system_darwin.go
index e0ece7662..b49f9ae60 100644
--- a/pkg/fs/local_file_system_darwin.go
+++ b/pkg/fs/local_file_system_darwin.go
@@ -84,6 +84,23 @@ func syncFile(file *os.File) error {
return nil
}
+// syncDir opens the directory at path, fsyncs it, and closes it. Used by
+// WriteAtomic to make a rename durable on POSIX filesystems. Returns the
+// underlying error rather than panicking so callers can wrap it in a
+// FileSystemError. The Windows implementation is a no-op (NTFS does not
+// expose directory fsync; os.Rename is atomic on its own).
+func syncDir(path string) error {
+ dir, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ if syncErr := dir.Sync(); syncErr != nil {
+ _ = dir.Close()
+ return syncErr
+ }
+ return dir.Close()
+}
+
func mustGetFileStat(path string) (*syscall.Stat_t, error) {
fi, err := os.Stat(path)
if err != nil {
diff --git a/pkg/fs/local_file_system_linux.go b/pkg/fs/local_file_system_linux.go
index c660b4915..1ace320ad 100644
--- a/pkg/fs/local_file_system_linux.go
+++ b/pkg/fs/local_file_system_linux.go
@@ -84,6 +84,23 @@ func syncFile(file *os.File) error {
return nil
}
+// syncDir opens the directory at path, fsyncs it, and closes it. Used by
+// WriteAtomic to make a rename durable on POSIX filesystems. Returns the
+// underlying error rather than panicking so callers can wrap it in a
+// FileSystemError. The Windows implementation is a no-op (NTFS does not
+// expose directory fsync; os.Rename is atomic on its own).
+func syncDir(path string) error {
+ dir, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ if syncErr := dir.Sync(); syncErr != nil {
+ _ = dir.Close()
+ return syncErr
+ }
+ return dir.Close()
+}
+
func mustGetFileStat(path string) (*syscall.Stat_t, error) {
fi, err := os.Stat(path)
if err != nil {
diff --git a/pkg/fs/local_file_system_test.go b/pkg/fs/local_file_system_test.go
index b96dc9dc4..d97012da1 100644
--- a/pkg/fs/local_file_system_test.go
+++ b/pkg/fs/local_file_system_test.go
@@ -307,6 +307,68 @@ var _ = ginkgo.Describe("Local File System", func() {
_, err = os.Stat(fileName)
gomega.Expect(err).To(gomega.HaveOccurred())
})
+
+ ginkgo.It("SeqWriter Close propagates sync error", func() {
+ sw := file.SequentialWrite()
+ // Force the underlying *os.File closed before sw.Close() runs so
+ // SyncAndDropCache (or the bufio Flush) sees a bad fd. The fix
+ // requires the resulting error to surface from sw.Close() instead
+ // of being silently dropped.
+ localFile, ok := file.(*LocalFile)
+ gomega.Expect(ok).To(gomega.BeTrue())
+ gomega.Expect(localFile.file.Close()).To(gomega.Succeed())
+ closeErr := sw.Close()
+ gomega.Expect(closeErr).To(gomega.HaveOccurred())
+ })
+
+ ginkgo.It("WriteAtomic leaves no .tmp on success", func() {
+ target := filepath.Join(dirName, "metadata.json")
+ n, err := fs.WriteAtomic([]byte(`{"v":1}`), target, 0o600)
+ gomega.Expect(err).ToNot(gomega.HaveOccurred())
+ gomega.Expect(n).To(gomega.Equal(7))
+ final, readErr := os.ReadFile(target)
+ gomega.Expect(readErr).ToNot(gomega.HaveOccurred())
+ gomega.Expect(string(final)).To(gomega.Equal(`{"v":1}`))
+ _, statErr := os.Stat(target + ".tmp")
+ gomega.Expect(os.IsNotExist(statErr)).To(gomega.BeTrue())
+ })
+
+ ginkgo.It("WriteAtomic leaves only .tmp on a forged crash", func() {
+ target := filepath.Join(dirName, "metadata.json")
+ prev := testHookAfterTmpFsync
+ testHookAfterTmpFsync = func() { panic("simulated crash") }
+ defer func() {
+ testHookAfterTmpFsync = prev
+ recovered := recover()
+ gomega.Expect(recovered).ToNot(gomega.BeNil(), "hook should have caused a panic")
+ // After the forged crash, .tmp must remain (was fsynced) and
+ // the final file must not exist (rename never ran).
+ _, statTmp := os.Stat(target + ".tmp")
+ gomega.Expect(statTmp).ToNot(gomega.HaveOccurred(), "tmp must remain after crash")
+ _, statFinal := os.Stat(target)
+ gomega.Expect(os.IsNotExist(statFinal)).To(gomega.BeTrue(), "final must not exist if rename never ran")
+ }()
+ _, _ = fs.WriteAtomic([]byte("data"), target, 0o600)
+ })
+
+ ginkgo.It("CleanupLeftoverTmp removes .tmp when the final file exists", func() {
+ final := filepath.Join(dirName, "metadata.json")
+ gomega.Expect(os.WriteFile(final, []byte("ok"), 0o600)).To(gomega.Succeed())
+ gomega.Expect(os.WriteFile(final+".tmp", []byte("stale"), 0o600)).To(gomega.Succeed())
+ CleanupLeftoverTmp(fs, dirName)
+ _, statErr := os.Stat(final + ".tmp")
+ gomega.Expect(os.IsNotExist(statErr)).To(gomega.BeTrue())
+ _, statFinal := os.Stat(final)
+ gomega.Expect(statFinal).ToNot(gomega.HaveOccurred())
+ })
+
+ ginkgo.It("CleanupLeftoverTmp leaves .tmp when no final exists", func() {
+ final := filepath.Join(dirName, "metadata.json")
+ gomega.Expect(os.WriteFile(final+".tmp", []byte("stranded"), 0o600)).To(gomega.Succeed())
+ CleanupLeftoverTmp(fs, dirName)
+ _, statErr := os.Stat(final + ".tmp")
+ gomega.Expect(statErr).ToNot(gomega.HaveOccurred())
+ })
})
ginkgo.Context("Hard Link Operations", func() {
diff --git a/pkg/fs/local_file_system_testhook_test.go b/pkg/fs/local_file_system_testhook_test.go
new file mode 100644
index 000000000..8fe8cfff5
--- /dev/null
+++ b/pkg/fs/local_file_system_testhook_test.go
@@ -0,0 +1,29 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package fs
+
+// testHookAfterTmpFsync is overridden by tests to simulate a crash between
+// WriteAtomic's fsync(tmp) and rename. Because this file ends in _test.go,
+// the indirection is removed at production link time — production binaries
+// see only the no-op invokeTestHookAfterTmpFsync defined in
+// local_file_system.go.
+var testHookAfterTmpFsync = func() {}
+
+func init() {
+ invokeTestHookAfterTmpFsync = func() { testHookAfterTmpFsync() }
+}
diff --git a/pkg/fs/local_file_system_windows.go b/pkg/fs/local_file_system_windows.go
index ad7a5131e..80290af28 100644
--- a/pkg/fs/local_file_system_windows.go
+++ b/pkg/fs/local_file_system_windows.go
@@ -66,6 +66,12 @@ func syncFile(_ *os.File) error {
return nil
}
+// syncDir is a no-op on Windows. NTFS does not expose directory fsync via
+// os.File.Sync the way ext4/xfs/apfs do; os.Rename on NTFS is atomic on its
+// own and the existing SyncPath has the same no-op pattern. Linux/Darwin
+// implementations open the dir and call file.Sync.
+func syncDir(_ string) error { return nil }
+
func CompareINode(srcPath, destPath string) error {
return nil
}
(skywalking-banyandb) 08/14: Fix FODC lifecycle endpoint wrong encoding and increase the gRPC timeout between agent and proxy (#1103)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 5cfe59a124779b5cbb3fdfadcf4066809fa183c5
Author: mrproliu <74...@qq.com>
AuthorDate: Thu Apr 30 10:11:53 2026 +0800
Fix FODC lifecycle endpoint wrong encoding and increase the gRPC timeout between agent and proxy (#1103)
---
CHANGES.md | 2 +
fodc/agent/internal/lifecycle/collector.go | 56 +++--
fodc/agent/internal/lifecycle/collector_test.go | 263 ++++++++++++++++++++++--
fodc/internal/timeouts/timeouts.go | 34 +++
fodc/proxy/internal/api/server.go | 66 +++++-
fodc/proxy/internal/api/server_test.go | 196 ++++++++++++++++++
fodc/proxy/internal/lifecycle/manager.go | 9 +-
7 files changed, 584 insertions(+), 42 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index d5b5ab50d..6356ba45e 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -29,6 +29,8 @@ Release Notes.
- Avoid FODC lifecycle inspection failing on busy data nodes by raising the per-broadcast `CollectDataInfo` / `CollectLiaisonInfo` deadline from 5s to 30s and parallelizing per-group inspection in the cluster-internal `InspectAll`.
- Fix flaky `file_snapshot` subtest in measure/stream/trace by waiting until every introduced mem part has been flushed to disk, instead of only checking the latest snapshot creator.
- Fix flaky `TestCollectWithPartialClosedSegments` by raising `SegmentIdleTimeout` so wall-clock variance on slow CI does not mark still-open segments as idle.
+- Fix FODC lifecycle cache poisoning where transient `InspectAll` failures were cached for 10 minutes and masked liaison recovery; raise FODC agent and proxy timeouts from 10s to 40s.
+- Fix FODC `/cluster/lifecycle` dropping zero-valued group fields (e.g. `replicas=0`, `close=false`) under `encoding/json` + `omitempty`; switch to `protojson` so all fields are emitted (nil nested messages serialize as `null`).
### Chores
diff --git a/fodc/agent/internal/lifecycle/collector.go b/fodc/agent/internal/lifecycle/collector.go
index 6b9c559ff..1f94636dc 100644
--- a/fodc/agent/internal/lifecycle/collector.go
+++ b/fodc/agent/internal/lifecycle/collector.go
@@ -20,6 +20,7 @@ package lifecycle
import (
"context"
+ "fmt"
"os"
"path/filepath"
"sort"
@@ -34,6 +35,7 @@ import (
"google.golang.org/grpc/status"
fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
+ "github.com/apache/skywalking-banyandb/fodc/internal/timeouts"
"github.com/apache/skywalking-banyandb/pkg/logger"
)
@@ -44,8 +46,6 @@ const (
maxReportFileSize = 5 * 1024 * 1024 // 5MB
)
-const grpcTimeout = 10 * time.Second
-
// Collector collects lifecycle data from local files.
type Collector struct {
lastCollectTime time.Time
@@ -73,8 +73,10 @@ func NewCollector(log *logger.Logger, grpcAddr, reportDir string, cacheTTL time.
}
}
-// Collect collects lifecycle data from local files.
-// Data is cached for cacheTTL duration; after expiry the next call refreshes the cache.
+// Collect returns lifecycle data, serving the cache when fresh. A transient InspectAll
+// failure is propagated to the caller (the cache is left untouched so the next call
+// retries) so that callers can distinguish a real failure from a healthy liaison that
+// happens to have zero groups.
func (c *Collector) Collect(ctx context.Context) (*fodcv1.LifecycleData, error) {
c.mu.RLock()
if c.currentData != nil && c.cacheTTL > 0 && c.nowFunc().Sub(c.lastCollectTime) < c.cacheTTL {
@@ -84,9 +86,19 @@ func (c *Collector) Collect(ctx context.Context) (*fodcv1.LifecycleData, error)
}
c.mu.RUnlock()
+ reports := c.readReportFiles()
+ groups, err := c.collectGroups(ctx)
+ if err != nil {
+ if c.log != nil {
+ c.log.Warn().Err(err).Int("reports", len(reports)).
+ Msg("InspectAll failed; cache untouched, propagating error to caller")
+ }
+ return nil, err
+ }
+
data := &fodcv1.LifecycleData{
- Reports: c.readReportFiles(),
- Groups: c.collectGroups(ctx),
+ Reports: reports,
+ Groups: groups,
}
c.mu.Lock()
c.currentData = data
@@ -95,23 +107,28 @@ func (c *Collector) Collect(ctx context.Context) (*fodcv1.LifecycleData, error)
return data, nil
}
-func (c *Collector) collectGroups(ctx context.Context) []*fodcv1.GroupLifecycleInfo {
+// collectGroups invokes InspectAll on the local liaison.
+//
+// Return contract:
+// - (nil, nil): no RPC was issued (grpcAddr empty, or InspectAll already known to be Unimplemented).
+// Callers treat this as a successful collection with no groups.
+// - (non-nil slice, nil): RPC succeeded. The slice is empty (non-nil) when the server returned no groups.
+// - (nil, err): a transient error occurred (DeadlineExceeded, Unavailable, dial failure, etc.).
+// Callers must NOT cache this outcome.
+func (c *Collector) collectGroups(ctx context.Context) ([]*fodcv1.GroupLifecycleInfo, error) {
if c.grpcAddr == "" || c.grpcUnimplemented.Load() {
- return nil
+ return nil, nil
}
conn, err := grpc.NewClient(c.grpcAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
- if c.log != nil {
- c.log.Warn().Err(err).Str("addr", c.grpcAddr).Msg("Failed to create gRPC connection for InspectAll")
- }
- return nil
+ return nil, fmt.Errorf("dial %s: %w", c.grpcAddr, err)
}
defer func() {
_ = conn.Close()
}()
client := fodcv1.NewGroupLifecycleServiceClient(conn)
- reqCtx, cancel := context.WithTimeout(ctx, grpcTimeout)
+ reqCtx, cancel := context.WithTimeout(ctx, timeouts.AgentInspectAll)
defer cancel()
resp, err := client.InspectAll(reqCtx, &fodcv1.InspectAllRequest{})
if err != nil {
@@ -120,14 +137,15 @@ func (c *Collector) collectGroups(ctx context.Context) []*fodcv1.GroupLifecycleI
if c.log != nil {
c.log.Info().Str("addr", c.grpcAddr).Msg("GroupLifecycleService.InspectAll is not implemented, skipping future calls")
}
- return nil
+ return nil, nil
}
- if c.log != nil {
- c.log.Warn().Err(err).Str("addr", c.grpcAddr).Msg("InspectAll failed, skipping groups")
- }
- return nil
+ return nil, fmt.Errorf("InspectAll on %s: %w", c.grpcAddr, err)
+ }
+ got := resp.GetGroups()
+ if got == nil {
+ return []*fodcv1.GroupLifecycleInfo{}, nil
}
- return resp.GetGroups()
+ return got, nil
}
func (c *Collector) readReportFiles() []*fodcv1.LifecycleReport {
diff --git a/fodc/agent/internal/lifecycle/collector_test.go b/fodc/agent/internal/lifecycle/collector_test.go
index 0be13c09a..ab44d5053 100644
--- a/fodc/agent/internal/lifecycle/collector_test.go
+++ b/fodc/agent/internal/lifecycle/collector_test.go
@@ -18,14 +18,21 @@
package lifecycle
import (
+ "context"
+ "net"
"os"
"path/filepath"
+ "sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+ fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
"github.com/apache/skywalking-banyandb/pkg/logger"
)
@@ -36,6 +43,73 @@ func initTestLogger(t *testing.T) *logger.Logger {
return logger.GetLogger("test", "lifecycle")
}
+// fakeLifecycleService is a programmable test double for GroupLifecycleService.InspectAll.
+// Each registered behavior is consumed in order; the call counter lets tests assert how
+// many times Collect actually reached the gRPC server (i.e. whether the cache served the
+// request without an RPC).
+type fakeLifecycleService struct {
+ fodcv1.UnimplementedGroupLifecycleServiceServer
+ behaviors []func() (*fodcv1.InspectAllResponse, error)
+ mu sync.Mutex
+ callCount int
+}
+
+func (f *fakeLifecycleService) InspectAll(_ context.Context, _ *fodcv1.InspectAllRequest) (*fodcv1.InspectAllResponse, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ if f.callCount >= len(f.behaviors) {
+ f.callCount++
+ return nil, status.Error(codes.Internal, "fakeLifecycleService: behavior list exhausted")
+ }
+ behavior := f.behaviors[f.callCount]
+ f.callCount++
+ return behavior()
+}
+
+func (f *fakeLifecycleService) calls() int {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return f.callCount
+}
+
+func returnGroups(groups []*fodcv1.GroupLifecycleInfo) func() (*fodcv1.InspectAllResponse, error) {
+ return func() (*fodcv1.InspectAllResponse, error) {
+ return &fodcv1.InspectAllResponse{Groups: groups}, nil
+ }
+}
+
+func returnError(code codes.Code, msg string) func() (*fodcv1.InspectAllResponse, error) {
+ return func() (*fodcv1.InspectAllResponse, error) {
+ return nil, status.Error(code, msg)
+ }
+}
+
+// startLocalServer spins up a real gRPC server on a random localhost port hosting the
+// supplied fake and returns the listen address. The server is torn down via t.Cleanup.
+func startLocalServer(t *testing.T, fake *fakeLifecycleService) string {
+ t.Helper()
+ lis, err := net.Listen("tcp", "127.0.0.1:0")
+ require.NoError(t, err)
+ srv := grpc.NewServer()
+ fodcv1.RegisterGroupLifecycleServiceServer(srv, fake)
+ go func() {
+ _ = srv.Serve(lis)
+ }()
+ t.Cleanup(srv.Stop)
+ return lis.Addr().String()
+}
+
+// newCollectorForFake constructs a Collector wired to a fresh local gRPC server backed by fake.
+func newCollectorForFake(t *testing.T, fake *fakeLifecycleService, cacheTTL time.Duration) *Collector {
+ t.Helper()
+ addr := startLocalServer(t, fake)
+ return NewCollector(initTestLogger(t), addr, t.TempDir(), cacheTTL)
+}
+
+func sampleGroup(name string) *fodcv1.GroupLifecycleInfo {
+ return &fodcv1.GroupLifecycleInfo{Name: name}
+}
+
func TestNewCollector(t *testing.T) {
log := initTestLogger(t)
collector := NewCollector(log, "", "", 10*time.Minute)
@@ -51,11 +125,10 @@ func TestCollector_ReadReportFiles_EmptyDir(t *testing.T) {
assert.Empty(t, reports)
}
-func TestCollector_Collect(t *testing.T) {
+func TestCollector_Collect_NoGRPC(t *testing.T) {
log := initTestLogger(t)
collector := NewCollector(log, "", "", 0)
- ctx := t.Context()
- data, err := collector.Collect(ctx)
+ data, err := collector.Collect(t.Context())
require.NoError(t, err)
require.NotNil(t, data)
assert.NotNil(t, collector.currentData)
@@ -88,20 +161,19 @@ func TestCollector_CacheTTL(t *testing.T) {
assert.NotSame(t, data1, data3)
}
-func TestCollector_GrpcUnimplemented_SkipsFutureCalls(t *testing.T) {
+func TestCollector_ZeroTTL_AlwaysRefreshes(t *testing.T) {
log := initTestLogger(t)
- collector := NewCollector(log, "localhost:0", "", 0)
-
- assert.False(t, collector.grpcUnimplemented.Load())
+ collector := NewCollector(log, "", "", 0)
+ ctx := t.Context()
- groups := collector.collectGroups(t.Context())
- assert.Nil(t, groups)
- assert.False(t, collector.grpcUnimplemented.Load())
+ data1, err := collector.Collect(ctx)
+ require.NoError(t, err)
+ require.NotNil(t, data1)
- collector.grpcUnimplemented.Store(true)
- groups = collector.collectGroups(t.Context())
- assert.Nil(t, groups)
- assert.True(t, collector.grpcUnimplemented.Load())
+ data2, err := collector.Collect(ctx)
+ require.NoError(t, err)
+ require.NotNil(t, data2)
+ assert.NotSame(t, data1, data2)
}
func TestCollector_ReadReportFiles_MaxFiles(t *testing.T) {
@@ -109,7 +181,6 @@ func TestCollector_ReadReportFiles_MaxFiles(t *testing.T) {
dir := t.TempDir()
collector := NewCollector(log, "", dir, 0)
- // Create 7 files with timestamp-like names so sort order is predictable
for idx, name := range []string{
"2026-03-20.json",
"2026-03-21.json",
@@ -165,18 +236,172 @@ func TestCollector_ReadReportFiles_OversizedFile(t *testing.T) {
assert.Equal(t, "2026-03-26.json", reports[0].Filename)
}
-func TestCollector_ZeroTTL_AlwaysRefreshes(t *testing.T) {
+func TestCollectGroups_NoGRPCAddrShortCircuits(t *testing.T) {
log := initTestLogger(t)
collector := NewCollector(log, "", "", 0)
+ groups, err := collector.collectGroups(t.Context())
+ require.NoError(t, err)
+ assert.Nil(t, groups)
+}
+
+func TestCollectGroups_UnimplementedSetsFlagAndCachesNil(t *testing.T) {
+ fake := &fakeLifecycleService{
+ behaviors: []func() (*fodcv1.InspectAllResponse, error){
+ returnError(codes.Unimplemented, "not supported"),
+ },
+ }
+ collector := newCollectorForFake(t, fake, time.Minute)
+
+ groups, err := collector.collectGroups(t.Context())
+ require.NoError(t, err)
+ assert.Nil(t, groups)
+ assert.True(t, collector.grpcUnimplemented.Load())
+
+ // Subsequent direct calls short-circuit without touching the server.
+ groups2, err := collector.collectGroups(t.Context())
+ require.NoError(t, err)
+ assert.Nil(t, groups2)
+ assert.Equal(t, 1, fake.calls())
+}
+
+func TestCollectGroups_NilResponseGroupsBecomesEmptySlice(t *testing.T) {
+ fake := &fakeLifecycleService{
+ behaviors: []func() (*fodcv1.InspectAllResponse, error){
+ returnGroups(nil),
+ },
+ }
+ collector := newCollectorForFake(t, fake, time.Minute)
+
+ groups, err := collector.collectGroups(t.Context())
+ require.NoError(t, err)
+ require.NotNil(t, groups, "successful InspectAll with empty groups must return non-nil empty slice, not nil")
+ assert.Empty(t, groups)
+}
+
+func TestCollect_CachesSuccessfulResult(t *testing.T) {
+ fake := &fakeLifecycleService{
+ behaviors: []func() (*fodcv1.InspectAllResponse, error){
+ returnGroups([]*fodcv1.GroupLifecycleInfo{sampleGroup("g1"), sampleGroup("g2")}),
+ },
+ }
+ collector := newCollectorForFake(t, fake, time.Minute)
ctx := t.Context()
data1, err := collector.Collect(ctx)
require.NoError(t, err)
- require.NotNil(t, data1)
+ require.Len(t, data1.Groups, 2)
// With zero TTL, every call should re-collect
data2, err := collector.Collect(ctx)
require.NoError(t, err)
- require.NotNil(t, data2)
- assert.NotSame(t, data1, data2)
+ assert.Same(t, data1, data2, "second Collect within TTL must return the cached pointer")
+ assert.Equal(t, 1, fake.calls(), "InspectAll must not be invoked on cache hit")
+}
+
+func TestCollect_DoesNotCacheTransientFailure(t *testing.T) {
+ fake := &fakeLifecycleService{
+ behaviors: []func() (*fodcv1.InspectAllResponse, error){
+ returnError(codes.DeadlineExceeded, "simulated timeout"),
+ returnGroups([]*fodcv1.GroupLifecycleInfo{sampleGroup("g1")}),
+ },
+ }
+ collector := newCollectorForFake(t, fake, 10*time.Minute)
+ ctx := t.Context()
+
+ data1, err := collector.Collect(ctx)
+ require.Error(t, err, "transient failure must propagate to caller")
+ assert.Nil(t, data1, "no LifecycleData on failure")
+
+ collector.mu.RLock()
+ cachedData := collector.currentData
+ cachedTime := collector.lastCollectTime
+ collector.mu.RUnlock()
+ assert.Nil(t, cachedData, "transient failure must not write to cache")
+ assert.True(t, cachedTime.IsZero(), "lastCollectTime must remain zero after transient failure")
+
+ data2, err := collector.Collect(ctx)
+ require.NoError(t, err)
+ require.Len(t, data2.Groups, 1, "next call must retry InspectAll, not return stale empty")
+ assert.Equal(t, "g1", data2.Groups[0].Name)
+ assert.Equal(t, 2, fake.calls(), "InspectAll must be invoked twice")
+}
+
+func TestCollect_FailurePropagatesError(t *testing.T) {
+ successGroups := []*fodcv1.GroupLifecycleInfo{sampleGroup("g1"), sampleGroup("g2"), sampleGroup("g3")}
+ fake := &fakeLifecycleService{
+ behaviors: []func() (*fodcv1.InspectAllResponse, error){
+ returnGroups(successGroups),
+ returnError(codes.DeadlineExceeded, "simulated timeout"),
+ },
+ }
+ collector := newCollectorForFake(t, fake, 0) // zero TTL forces both calls to hit the server
+ ctx := t.Context()
+
+ data1, err := collector.Collect(ctx)
+ require.NoError(t, err)
+ require.Len(t, data1.Groups, 3)
+
+ data2, err := collector.Collect(ctx)
+ require.Error(t, err, "transient failure after a successful collect must still surface as an error")
+ assert.Nil(t, data2, "caller must not receive a LifecycleData payload on failure")
+ assert.Equal(t, 2, fake.calls())
+}
+
+func TestCollect_TransientFailureDoesNotAdvanceLastCollectTime(t *testing.T) {
+ fake := &fakeLifecycleService{
+ behaviors: []func() (*fodcv1.InspectAllResponse, error){
+ returnGroups([]*fodcv1.GroupLifecycleInfo{sampleGroup("g1")}),
+ returnError(codes.Unavailable, "simulated outage"),
+ returnGroups([]*fodcv1.GroupLifecycleInfo{sampleGroup("g1"), sampleGroup("g2")}),
+ },
+ }
+ collector := newCollectorForFake(t, fake, time.Hour)
+ ctx := t.Context()
+
+ now := time.Unix(1_000_000_000, 0)
+ collector.nowFunc = func() time.Time { return now }
+
+ _, err := collector.Collect(ctx)
+ require.NoError(t, err)
+ collector.mu.RLock()
+ successTime := collector.lastCollectTime
+ collector.mu.RUnlock()
+ assert.Equal(t, now, successTime)
+
+ now = now.Add(2 * time.Hour) // expire the cache to force a new RPC
+ _, err = collector.Collect(ctx)
+ require.Error(t, err, "transient failure must surface as an error to the caller")
+ collector.mu.RLock()
+ failureTime := collector.lastCollectTime
+ collector.mu.RUnlock()
+ assert.Equal(t, successTime, failureTime, "transient failure must not advance lastCollectTime")
+
+ data, err := collector.Collect(ctx)
+ require.NoError(t, err)
+ require.Len(t, data.Groups, 2, "after recovery, fresh groups must be cached")
+ assert.Equal(t, 3, fake.calls())
+}
+
+func TestCollect_UnimplementedCachesAndStops(t *testing.T) {
+ fake := &fakeLifecycleService{
+ behaviors: []func() (*fodcv1.InspectAllResponse, error){
+ returnError(codes.Unimplemented, "not supported"),
+ },
+ }
+ collector := newCollectorForFake(t, fake, 10*time.Minute)
+ ctx := t.Context()
+
+ data1, err := collector.Collect(ctx)
+ require.NoError(t, err)
+ assert.Nil(t, data1.Groups)
+ assert.True(t, collector.grpcUnimplemented.Load())
+
+ collector.mu.RLock()
+ require.NotNil(t, collector.currentData, "Unimplemented is a known terminal state and must be cached")
+ collector.mu.RUnlock()
+
+ data2, err := collector.Collect(ctx)
+ require.NoError(t, err)
+ assert.Same(t, data1, data2, "subsequent calls within TTL must hit cache")
+ assert.Equal(t, 1, fake.calls(), "Unimplemented must short-circuit subsequent RPCs")
}
diff --git a/fodc/internal/timeouts/timeouts.go b/fodc/internal/timeouts/timeouts.go
new file mode 100644
index 000000000..08d9e958c
--- /dev/null
+++ b/fodc/internal/timeouts/timeouts.go
@@ -0,0 +1,34 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Package timeouts holds timeout constants shared between the FODC agent and proxy.
+// Co-locating these constants ensures the proxy-side per-agent collection deadline is
+// always strictly greater than the agent-side InspectAll timeout, regardless of who
+// imports them.
+package timeouts
+
+import "time"
+
+// AgentInspectAll bounds how long a single InspectAll call against the local liaison
+// may run on the FODC agent side.
+const AgentInspectAll = 40 * time.Second
+
+// ProxySlack is the additional time the proxy waits, on top of AgentInspectAll, before
+// declaring an agent unresponsive. It must be strictly greater than zero so the proxy
+// always outlasts the agent's own deadline and never gives up while a slow but still
+// progressing InspectAll call is in flight.
+const ProxySlack = 10 * time.Second
diff --git a/fodc/proxy/internal/api/server.go b/fodc/proxy/internal/api/server.go
index 3ff5de920..f8c9338fa 100644
--- a/fodc/proxy/internal/api/server.go
+++ b/fodc/proxy/internal/api/server.go
@@ -28,6 +28,9 @@ import (
"strings"
"time"
+ "google.golang.org/protobuf/encoding/protojson"
+
+ fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/cluster"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/lifecycle"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/metrics"
@@ -35,6 +38,15 @@ import (
"github.com/apache/skywalking-banyandb/pkg/logger"
)
+// lifecycleGroupMarshaler emits zero-value protobuf fields (replicas=0, close=false, empty
+// stages, etc.) as explicit JSON keys instead of dropping them. The default encoding/json +
+// protoc-gen-go combination would silence those fields via `omitempty`, leaving downstream
+// consumers (SRE agent) unable to tell "field absent" from "value is zero".
+var lifecycleGroupMarshaler = protojson.MarshalOptions{
+ UseProtoNames: true,
+ EmitUnpopulated: true,
+}
+
// Server exposes REST and Prometheus-style endpoints for external consumption.
type Server struct {
metricsAggregator *metrics.Aggregator
@@ -496,9 +508,23 @@ func (s *Server) handleClusterLifecycle(w http.ResponseWriter, r *http.Request)
}
lifecycleData, agentSummary := s.lifecycleManager.CollectLifecycle(r.Context())
+ groupsJSON, err := marshalLifecycleGroups(lifecycleData.Groups)
+ if err != nil {
+ s.logger.Error().Err(err).Msg("Failed to marshal lifecycle groups")
+ http.Error(w, "Failed to serialize lifecycle groups", http.StatusInternalServerError)
+ return
+ }
+
+ statusesJSON, err := marshalLifecycleStatuses(lifecycleData.LifecycleStatuses)
+ if err != nil {
+ s.logger.Error().Err(err).Msg("Failed to marshal lifecycle statuses")
+ http.Error(w, "Failed to serialize lifecycle statuses", http.StatusInternalServerError)
+ return
+ }
+
body := map[string]interface{}{
- "groups": lifecycleData.Groups,
- "lifecycle_statuses": lifecycleData.LifecycleStatuses,
+ "groups": groupsJSON,
+ "lifecycle_statuses": statusesJSON,
"agent_summary": agentSummary,
}
switch {
@@ -519,6 +545,42 @@ func (s *Server) handleClusterLifecycle(w http.ResponseWriter, r *http.Request)
}
}
+func marshalLifecycleGroups(groups []*fodcv1.GroupLifecycleInfo) ([]json.RawMessage, error) {
+ out := make([]json.RawMessage, 0, len(groups))
+ for _, g := range groups {
+ raw, err := lifecycleGroupMarshaler.Marshal(g)
+ if err != nil {
+ return nil, fmt.Errorf("marshal group %q: %w", g.GetName(), err)
+ }
+ out = append(out, raw)
+ }
+ return out, nil
+}
+
+// lifecycleStatusJSON mirrors lifecycle.PodLifecycleStatus but holds each report as a
+// pre-marshaled protojson blob so the LifecycleReport's protobuf zero-value fields are
+// emitted consistently with the groups payload.
+type lifecycleStatusJSON struct {
+ PodName string `json:"pod_name"`
+ Reports []json.RawMessage `json:"reports"`
+}
+
+func marshalLifecycleStatuses(statuses []*lifecycle.PodLifecycleStatus) ([]lifecycleStatusJSON, error) {
+ out := make([]lifecycleStatusJSON, 0, len(statuses))
+ for _, s := range statuses {
+ reports := make([]json.RawMessage, 0, len(s.Reports))
+ for _, r := range s.Reports {
+ raw, err := lifecycleGroupMarshaler.Marshal(r)
+ if err != nil {
+ return nil, fmt.Errorf("marshal report %q: %w", r.GetFilename(), err)
+ }
+ reports = append(reports, raw)
+ }
+ out = append(out, lifecycleStatusJSON{PodName: s.PodName, Reports: reports})
+ }
+ return out, nil
+}
+
// handleClusterTopology handles GET /cluster/topology endpoint.
func (s *Server) handleClusterTopology(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
diff --git a/fodc/proxy/internal/api/server_test.go b/fodc/proxy/internal/api/server_test.go
index b39d8c609..a4cf4ba1f 100644
--- a/fodc/proxy/internal/api/server_test.go
+++ b/fodc/proxy/internal/api/server_test.go
@@ -833,3 +833,199 @@ func TestHandleClusterLifecycle_NoAgents_FlagsErrorInBody(t *testing.T) {
assert.EqualValues(t, 0, summary["total"], "agent_summary.total should be 0 when no agents are registered")
assert.EqualValues(t, 0, summary["responded"], "agent_summary.responded should be 0 when no agents are registered")
}
+
+// runLifecycleHandlerWithGroup wires up a lifecycle manager backed by a single agent that
+// reports the supplied group, invokes the HTTP handler, and returns the parsed response
+// body so callers can inspect raw JSON keys.
+func runLifecycleHandlerWithGroup(t *testing.T, group *fodcv1.GroupLifecycleInfo) map[string]interface{} {
+ t.Helper()
+ initTestLogger(t)
+ testLogger := logger.GetLogger("test", "api")
+ testRegistry := registry.NewAgentRegistry(testLogger, 5*time.Second, 10*time.Second, 100)
+ t.Cleanup(testRegistry.Stop)
+ mockSender := &mockRequestSender{}
+ aggregator := metrics.NewAggregator(testRegistry, mockSender, testLogger)
+
+ mockLifecycleRequester := &mockLifecycleDataRequester{
+ dataByAgent: make(map[string]*fodcv1.LifecycleData),
+ podByAgent: make(map[string]string),
+ }
+ lifecycleMgr := lifecycle.NewManager(testRegistry, mockLifecycleRequester, testLogger)
+ mockLifecycleRequester.lifecycleMgr = lifecycleMgr
+
+ identity := registry.AgentIdentity{
+ PodName: "test-pod-1",
+ Role: "datanode",
+ ContainerNames: []string{"banyandb"},
+ }
+ agentID, registerErr := testRegistry.RegisterAgent(context.Background(), identity)
+ require.NoError(t, registerErr)
+
+ mockLifecycleRequester.dataByAgent[agentID] = &fodcv1.LifecycleData{
+ Groups: []*fodcv1.GroupLifecycleInfo{group},
+ }
+ mockLifecycleRequester.podByAgent[agentID] = "test-pod-1"
+
+ server := NewServer(aggregator, nil, lifecycleMgr, testRegistry, testLogger)
+ req := httptest.NewRequest(http.MethodGet, "/cluster/lifecycle", nil)
+ resp := httptest.NewRecorder()
+ server.handleClusterLifecycle(resp, req)
+ require.Equal(t, http.StatusOK, resp.Code)
+
+ var body map[string]interface{}
+ require.NoError(t, json.NewDecoder(resp.Body).Decode(&body))
+ return body
+}
+
+// firstGroup pulls the first group from the decoded response body and asserts the basic shape.
+func firstGroup(t *testing.T, body map[string]interface{}) map[string]interface{} {
+ t.Helper()
+ groups, ok := body["groups"].([]interface{})
+ require.True(t, ok, "response body must contain a groups array")
+ require.NotEmpty(t, groups, "groups array must not be empty")
+ g, ok := groups[0].(map[string]interface{})
+ require.True(t, ok, "first group must decode to an object")
+ return g
+}
+
+func TestHandleClusterLifecycle_UsesSnakeCase(t *testing.T) {
+ group := &fodcv1.GroupLifecycleInfo{
+ Name: "g1",
+ ResourceOpts: &commonv1.ResourceOpts{
+ ShardNum: 4,
+ },
+ }
+ body := runLifecycleHandlerWithGroup(t, group)
+ g := firstGroup(t, body)
+
+ resourceOpts, ok := g["resource_opts"].(map[string]interface{})
+ require.True(t, ok, "field name must be snake_case 'resource_opts', not 'resourceOpts'")
+ assert.Equal(t, float64(4), resourceOpts["shard_num"], "field name must be snake_case 'shard_num'")
+ _, hasCamel := g["resourceOpts"]
+ assert.False(t, hasCamel, "camelCase key must not be emitted")
+}
+
+func TestHandleClusterLifecycle_EmitsZeroValueFields(t *testing.T) {
+ group := &fodcv1.GroupLifecycleInfo{
+ Name: "g1",
+ ResourceOpts: &commonv1.ResourceOpts{
+ ShardNum: 4,
+ Replicas: 0, // zero value must still appear in JSON output
+ Stages: []*commonv1.LifecycleStage{
+ {
+ Name: "warm",
+ ShardNum: 2,
+ Close: false, // zero value must still appear
+ Replicas: 0, // zero value must still appear
+ },
+ },
+ },
+ }
+ body := runLifecycleHandlerWithGroup(t, group)
+ g := firstGroup(t, body)
+
+ resourceOpts := g["resource_opts"].(map[string]interface{})
+ _, hasReplicas := resourceOpts["replicas"]
+ assert.True(t, hasReplicas, "zero-value replicas at ResourceOpts level must be emitted, not omitted")
+ assert.Equal(t, float64(0), resourceOpts["replicas"])
+
+ stages, ok := resourceOpts["stages"].([]interface{})
+ require.True(t, ok)
+ require.Len(t, stages, 1)
+ stage := stages[0].(map[string]interface{})
+
+ _, hasClose := stage["close"]
+ assert.True(t, hasClose, "zero-value close=false in LifecycleStage must be emitted")
+ assert.Equal(t, false, stage["close"])
+
+ _, hasStageReplicas := stage["replicas"]
+ assert.True(t, hasStageReplicas, "zero-value replicas in LifecycleStage must be emitted")
+ assert.Equal(t, float64(0), stage["replicas"])
+}
+
+func TestHandleClusterLifecycle_NilResourceOptsIsNull(t *testing.T) {
+ group := &fodcv1.GroupLifecycleInfo{
+ Name: "g1",
+ // ResourceOpts intentionally nil
+ }
+ body := runLifecycleHandlerWithGroup(t, group)
+ g := firstGroup(t, body)
+
+ require.Contains(t, g, "resource_opts", "EmitUnpopulated must surface the absent message as a key")
+ assert.Nil(t, g["resource_opts"], "nil protobuf message marshals to JSON null under EmitUnpopulated")
+}
+
+func TestHandleClusterLifecycle_DataInfoEmitted(t *testing.T) {
+ group := &fodcv1.GroupLifecycleInfo{
+ Name: "g1",
+ DataInfo: []*databasev1.DataInfo{
+ {DataSizeBytes: 1024},
+ },
+ }
+ body := runLifecycleHandlerWithGroup(t, group)
+ g := firstGroup(t, body)
+
+ dataInfo, ok := g["data_info"].([]interface{})
+ require.True(t, ok, "data_info must be encoded as snake_case array")
+ require.Len(t, dataInfo, 1)
+}
+
+func TestHandleClusterLifecycle_StatusReportsUseProtoJSON(t *testing.T) {
+ initTestLogger(t)
+ testLogger := logger.GetLogger("test", "api")
+ testRegistry := registry.NewAgentRegistry(testLogger, 5*time.Second, 10*time.Second, 100)
+ t.Cleanup(testRegistry.Stop)
+ mockSender := &mockRequestSender{}
+ aggregator := metrics.NewAggregator(testRegistry, mockSender, testLogger)
+
+ mockLifecycleRequester := &mockLifecycleDataRequester{
+ dataByAgent: make(map[string]*fodcv1.LifecycleData),
+ podByAgent: make(map[string]string),
+ }
+ lifecycleMgr := lifecycle.NewManager(testRegistry, mockLifecycleRequester, testLogger)
+ mockLifecycleRequester.lifecycleMgr = lifecycleMgr
+
+ identity := registry.AgentIdentity{
+ PodName: "test-pod-1",
+ Role: "datanode",
+ ContainerNames: []string{"banyandb"},
+ }
+ agentID, registerErr := testRegistry.RegisterAgent(context.Background(), identity)
+ require.NoError(t, registerErr)
+
+ mockLifecycleRequester.dataByAgent[agentID] = &fodcv1.LifecycleData{
+ Reports: []*fodcv1.LifecycleReport{
+ {Filename: "2026-04-29.json", ReportJson: `{"status":"ok"}`},
+ {Filename: "2026-04-28.json", ReportJson: ""}, // empty value must still be emitted
+ },
+ }
+ mockLifecycleRequester.podByAgent[agentID] = "test-pod-1"
+
+ server := NewServer(aggregator, nil, lifecycleMgr, testRegistry, testLogger)
+ req := httptest.NewRequest(http.MethodGet, "/cluster/lifecycle", nil)
+ resp := httptest.NewRecorder()
+ server.handleClusterLifecycle(resp, req)
+ require.Equal(t, http.StatusOK, resp.Code)
+
+ var body map[string]interface{}
+ require.NoError(t, json.NewDecoder(resp.Body).Decode(&body))
+
+ statuses, ok := body["lifecycle_statuses"].([]interface{})
+ require.True(t, ok)
+ require.Len(t, statuses, 1)
+ status := statuses[0].(map[string]interface{})
+ assert.Equal(t, "test-pod-1", status["pod_name"])
+
+ reports, ok := status["reports"].([]interface{})
+ require.True(t, ok)
+ require.Len(t, reports, 2)
+
+ first := reports[0].(map[string]interface{})
+ assert.Equal(t, "2026-04-29.json", first["filename"])
+ assert.Equal(t, `{"status":"ok"}`, first["report_json"], "field must be snake_case 'report_json'")
+
+ second := reports[1].(map[string]interface{})
+ _, hasReportJSON := second["report_json"]
+ assert.True(t, hasReportJSON, "empty report_json must still be emitted under EmitUnpopulated")
+ assert.Equal(t, "", second["report_json"])
+}
diff --git a/fodc/proxy/internal/lifecycle/manager.go b/fodc/proxy/internal/lifecycle/manager.go
index 207354730..23c7bc9db 100644
--- a/fodc/proxy/internal/lifecycle/manager.go
+++ b/fodc/proxy/internal/lifecycle/manager.go
@@ -21,14 +21,19 @@ package lifecycle
import (
"context"
"sync"
- "time"
fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
+ "github.com/apache/skywalking-banyandb/fodc/internal/timeouts"
"github.com/apache/skywalking-banyandb/fodc/proxy/internal/registry"
"github.com/apache/skywalking-banyandb/pkg/logger"
)
-const defaultCollectionTimeout = 10 * time.Second
+// defaultCollectionTimeout bounds how long the proxy waits for each agent to push back
+// its lifecycle data. It is derived from the agent-side InspectAll timeout plus a fixed
+// slack so that this deadline is strictly greater than the agent's own deadline; the
+// proxy must always outlast the agent and never give up while a still-progressing
+// InspectAll call is in flight on the agent side.
+const defaultCollectionTimeout = timeouts.AgentInspectAll + timeouts.ProxySlack
// PodLifecycleStatus represents lifecycle status for a single pod.
type PodLifecycleStatus struct {
(skywalking-banyandb) 04/14: Fix expired TLS test certificate (#1078)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit b98ee96bc2acb874e1bd11803ba278da20d08b47
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Mon Apr 20 14:11:32 2026 +0800
Fix expired TLS test certificate (#1078)
---
.gitignore | 3 --
CHANGES.md | 14 +++++++
.../schema/property/testdata/certs/server.crt | 47 +++++++++++-----------
3 files changed, 37 insertions(+), 27 deletions(-)
diff --git a/.gitignore b/.gitignore
index 3235cb1ac..9af8ace51 100644
--- a/.gitignore
+++ b/.gitignore
@@ -81,9 +81,6 @@ gomock_reflect*
# eBPF generated files and binaries
fodc/agent/internal/ktm/iomonitor/ebpf/generated/vmlinux.h
-<<<<<<< HEAD
# OpenSSL serial files
*.srl
-=======
->>>>>>> 29e87c9d (Stable Segment Endtime (#1051))
diff --git a/CHANGES.md b/CHANGES.md
index 26090e881..1a0884ddd 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -13,6 +13,20 @@ Release Notes.
- Fix property schema client connection not stable after data node restarted.
- Fix flaky on-disk integration tests caused by Ginkgo v2 random container shuffling closing gRPC connections prematurely.
- ui: fix query editor refresh/reset behavior and BydbQL keyword highlighting.
+- Disable the rotation task on warm and cold nodes to prevent incorrect segment boundaries during lifecycle migration.
+- Prevent epoch-dated segment directories (seg-19700101) from being created by zero timestamps in distributed sync paths.
+- Fix SIDX streaming sync sending SegmentID as MinTimestamp instead of the actual timestamp, causing sync failures on the receiving node.
+- Fix handoff controller TOCTOU race allowing disk size limit bypass, and populate sidx MinTimestamp/MaxTimestamp during replay to prevent corrupt segment creation on recovered nodes.
+- Delete orphaned parts when no snapshot references them during tsTable initialization.
+- Extract shared LocateAll on NodeRegistry to ensure resolveAssignments and syncer GetNodes always produce identical node lists, preventing liaison from enqueuing parts to online/healthy data nodes.
+- Add validation for MATCH and IN conditions in inverted index query builder, and handle nil OR branch when all entities are specific.
+- Fix wrong backup path of schema property.
+- Fix lifecycle migration failure when the target stage has `close: true`.
+
+### Chores
+
+- Upgrade Go and npm dependencies including etcd to v3.6.10, OpenTelemetry to v1.43.0, AWS SDK, and Google Cloud libraries.
+- Regenerate expired TLS test certificate with 100-year validity.
## 0.10.0
diff --git a/banyand/metadata/schema/property/testdata/certs/server.crt b/banyand/metadata/schema/property/testdata/certs/server.crt
index 9e213cf92..d5868ab0c 100644
--- a/banyand/metadata/schema/property/testdata/certs/server.crt
+++ b/banyand/metadata/schema/property/testdata/certs/server.crt
@@ -1,26 +1,25 @@
-----BEGIN CERTIFICATE-----
-MIIEUDCCAjigAwIBAgIUEG6jrP+X5pg7FNorVnQlGBIT1TYwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwITXlUZXN0Q0EwHhcNMjUwNDE3MTUxODU2WhcNMjYwNDE3
-MTUxODU2WjAjMSEwHwYDVQQDDBhXZXNhbXMtTWFjQm9vay1Qcm8ubG9jYWwwggEi
-MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDUGYdez3b3NGSwIliEJtEAINqt
-QpynwwUVIWkkuYqQEujX3M6s5N7xYP/7UCOV6OyZNbkt8GSXJb/03OvB/qVAKWXN
-Qx0wOKWSGxvK0wheXO3woIsLlrRYm5KeCiBCEhqu9wKDgQ2fy+ph3elNJMnayaJW
-GaizuvFEPYY52sae1iy/nyy976R5EHMWh6zafG7JuxEzAQaS04fB7IIMtcxA7XtF
-El00GtxVFVAGuUur9YLGDGRX0CR7z2eoqD04VoPzUMrP09y44LO0M1lIN5JbQW1p
-4Uojx5jWbBMz6wnK1gKEtAtyggJP0FYFES+f14/qoW8zNM7KKUcYe3SZoJ3lAgMB
-AAGjgYswgYgwRgYDVR0RBD8wPYIJbG9jYWxob3N0ghhXZXNhbXMtTWFjQm9vay1Q
-cm8ubG9jYWyHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAEwHQYDVR0OBBYEFLw6lkin
-ijLXLUJ+Rq+R8UD2865VMB8GA1UdIwQYMBaAFN9rOKK6q5ytlMd4Z2mre7snDNu3
-MA0GCSqGSIb3DQEBCwUAA4ICAQA9ptMjfg5ETA5eKxqXJ5AXxcFISmetSzJbAEsd
-c/toMfWxYSyt1+W6SDBI4S3vIHokh0uuK1K8/PiywQ6xQicNtKfY8R96ZP+ZcdsI
-Gj1VNWVY7FbbCaUHXX6oC5rpLrU7oeQJ0CEj+WNfPz9hfqRVz7WCHNIZGvJFdMLk
-Zm56vmXC4brTUSMasItgHt68iofWNBeeUpgyvhoAkzxDuJ7/5fp6BRo5p10Nb2qW
-FdYGnLiAyqX8bV/ZPjoUMWPn8Q3gtcfFhelEQqRiFdQyehjKkD1eIcgIgbJQokoo
-7SRqBEqdK5YDvd8+ZA8+O15nbAopRu+atWVoQaA0ho0KolzpqlsEvc4oTRpO5VAB
-4AA7RlrHmZamoBxlq+79tcbJ1277tFBxky/zT8TvuhmLUSYGzC9aLTsqpBuuHkm6
-am/AGAnwQsaVjie1FSTl7lNJkxSFpS6wUy/Xm4ynAtGtwUf1Nk0BreT4H11Fe42E
-jZrVYQruiU+qtchzwXnrOI8j1GPQHqbiZZPLgmo6TqFFJP9gCsrnqAKy0Fc3tzI8
-noTE8vTXQ2NNefMZz61Ad+8TETMGk/tQeOJG+EsJYb+Boj9t+1QGU4Wd9cI6jdX3
-1qp7DRBZFGzqVUgtXdgY2Dtuq7mpkANULqvD47iyhvnNY1XKGGfO1zgrbNXE7rCs
-pizyeQ==
+MIIEMjCCAhqgAwIBAgIUOLo1ww3AMbK+1IidApUXBT5sLNcwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwITXlUZXN0Q0EwIBcNMjYwNDIwMDMxMTAyWhgPMjEyNjAz
+MjcwMzExMDJaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBANQZh17Pdvc0ZLAiWIQm0QAg2q1CnKfDBRUhaSS5ipAS
+6Nfczqzk3vFg//tQI5Xo7Jk1uS3wZJclv/Tc68H+pUApZc1DHTA4pZIbG8rTCF5c
+7fCgiwuWtFibkp4KIEISGq73AoOBDZ/L6mHd6U0kydrJolYZqLO68UQ9hjnaxp7W
+LL+fLL3vpHkQcxaHrNp8bsm7ETMBBpLTh8Hsggy1zEDte0USXTQa3FUVUAa5S6v1
+gsYMZFfQJHvPZ6ioPThWg/NQys/T3Ljgs7QzWUg3kltBbWnhSiPHmNZsEzPrCcrW
+AoS0C3KCAk/QVgURL5/Xj+qhbzM0zsopRxh7dJmgneUCAwEAAaN7MHkwNwYDVR0R
+BDAwLoIJbG9jYWxob3N0gglsb2NhbGhvc3SHBH8AAAGHEAAAAAAAAAAAAAAAAAAA
+AAEwHQYDVR0OBBYEFLw6lkinijLXLUJ+Rq+R8UD2865VMB8GA1UdIwQYMBaAFN9r
+OKK6q5ytlMd4Z2mre7snDNu3MA0GCSqGSIb3DQEBCwUAA4ICAQB7LsMebHUcWQvS
+NDhjyB70UfGBPfktgoSDk7rwP360R+sqo4uIEqOnI2pZswL9JGrHvPwNlGFX/o8l
+CuvcsRnAndk9fu00kPrZ5PRf0pGIF6cHByb7p23KiF0RIkBSImJpZOP3UdQgRy8n
+YLzjQBAjL4rPXk7hcOuMmhsHlH3Xd9K6xCFYlLJWjsCVELqn8ptDcsrAmP9/hdGJ
+aQg2W8REKAfrn8zFi4/vd4+bx3FI4pAA05At6gU0pgnQJSlDffHbOQpaz8xVI/fR
+FICHT+LogddsjxBtSUE3vsGLsb6Ir2LIpgDSUaB42SUhMMZzkcMUwN+oGNWKtHrO
+ticJdF29ZzykCH/coRh8nmBltadkqwycPBKQsBPdoTnAfhTLE2EuGxE7kxKoPACe
+Jli0KVNw5D5ULGdTd22+3oKwmnXG1E02KwpbLTh9uxfJ72RzgFAhZ1ZYFotv73tE
+mHlQSdFy8bF8rJbCTOORBDgVLLMqlir7jIgKlfdEIc43jo/qXY98nCn1FpVwtXev
+ulUepTnmFu1mYbvki4MyVw1cMRybA5Bjy7QKGbGB6c/QciDV3VhprWCl0xwDY2y+
+lpnGjlrXkeFOwVi3r/yR85ZjgQxdePSSBjKN4bJ+6vUhBdeEvqIkvyJtB1I/H2/k
+WErUaS2Knu62xKbTQRnmk3ojH8vwIA==
-----END CERTIFICATE-----
(skywalking-banyandb) 07/14: fix: stabilize flaky TestCollectWithPartialClosedSegments and fit flaky-test in 50min (#1100)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 935db739e787cb36a26bc8f21e720035f4bcc996
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed Apr 29 08:57:07 2026 +0800
fix: stabilize flaky TestCollectWithPartialClosedSegments and fit flaky-test in 50min (#1100)
* fix(measure,stream,trace): wait until all mem parts flushed in file_snapshot tests
The file_snapshot wait loop only checked snapshot.creator, which races
with concurrent introductions: when flushTimeout=0 the flusher fires
per-part, so a flush of part N can flip creator to Flusher while part
N+1 is still a mem part. Closing then drops the still-in-memory part,
losing data on reopen.
Replace the creator check with a scan over snapshot.parts for any
partWrapper.mp != nil. Applied in measure/query_test.go (TestQueryResult
and TestQueryResult_QuotaExceeded), stream/block_scanner_test.go,
stream/query_by_idx_test.go, and trace/query_test.go.
* test(measure,stream,trace): use require.Eventually for file_snapshot wait loop
Replace the open-ended polling loop with require.Eventually, gated by a
30s deadline, so a stuck flusher (regression, deadlock, IO issue) fails
the test at the right line with a clear message instead of relying on
the global go-test timeout.
Extract the mem-part scan into per-package allPartsFlushed helpers so
the predicate is shared between TestQueryResult and the QuotaExceeded
variants in measure and stream. The trace package gets its own copy
because partWrapper.mp is package-private.
* chore(ci): set flaky-test --repeat to 0 to fit 50-min timeout
The hourly flaky-test job consistently hit the 50-min cap because
the integration suite needed ~25 min per iteration and --repeat 3
made it loop 4 times. Drop --repeat to 0 so one iteration runs and
completes inside the cap and the 1-hour cron interval.
* fix(storage): stabilize TestCollectWithPartialClosedSegments
Raise SegmentIdleTimeout from 100ms to 1h so wall-clock variance on slow
CI runners no longer flips still-open segments past the idle threshold.
The manual lastAccessed override is bumped to 2h ago so the segments the
test marks idle stay past the new threshold.
---
.github/workflows/flaky-test.yml | 2 +-
CHANGES.md | 2 ++
banyand/internal/storage/tsdb_test.go | 4 ++--
3 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/flaky-test.yml b/.github/workflows/flaky-test.yml
index f5e0b57c4..d2a8a8b9f 100644
--- a/.github/workflows/flaky-test.yml
+++ b/.github/workflows/flaky-test.yml
@@ -32,5 +32,5 @@ jobs:
uses: ./.github/workflows/test.yml
with:
test-name: Flaky Tests
- options: --vv --repeat 3 --label-filter '(integration&&!slow)||banyand'
+ options: --vv --repeat 0 --label-filter '(integration&&!slow)||banyand'
timeout-minutes: 50
diff --git a/CHANGES.md b/CHANGES.md
index eac054fab..d5b5ab50d 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -28,11 +28,13 @@ Release Notes.
- Fix lifecycle migration panic when a stream shard's snapshot has no element index (`idx/`) directory.
- Avoid FODC lifecycle inspection failing on busy data nodes by raising the per-broadcast `CollectDataInfo` / `CollectLiaisonInfo` deadline from 5s to 30s and parallelizing per-group inspection in the cluster-internal `InspectAll`.
- Fix flaky `file_snapshot` subtest in measure/stream/trace by waiting until every introduced mem part has been flushed to disk, instead of only checking the latest snapshot creator.
+- Fix flaky `TestCollectWithPartialClosedSegments` by raising `SegmentIdleTimeout` so wall-clock variance on slow CI does not mark still-open segments as idle.
### Chores
- Upgrade Go and npm dependencies including etcd to v3.6.10, OpenTelemetry to v1.43.0, AWS SDK, and Google Cloud libraries.
- Regenerate expired TLS test certificate with 100-year validity.
+- Set Ginkgo `--repeat` to 0 in the flaky-test workflow so the hourly run completes within the 50-minute timeout.
## 0.10.0
diff --git a/banyand/internal/storage/tsdb_test.go b/banyand/internal/storage/tsdb_test.go
index f07263741..6b786a9c5 100644
--- a/banyand/internal/storage/tsdb_test.go
+++ b/banyand/internal/storage/tsdb_test.go
@@ -575,7 +575,7 @@ func TestCollectWithPartialClosedSegments(t *testing.T) {
TTL: IntervalRule{Unit: DAY, Num: 7},
ShardNum: 2,
TSTableCreator: MockTSTableCreator,
- SegmentIdleTimeout: 100 * time.Millisecond, // Short idle timeout for testing
+ SegmentIdleTimeout: time.Hour, // Long enough that only segments with manually backdated lastAccessed are idle
}
ctx := context.Background()
@@ -627,7 +627,7 @@ func TestCollectWithPartialClosedSegments(t *testing.T) {
for _, s := range ss {
// Find segments we want to mark as idle (first and third)
if s.Start.Equal(segmentDates[0]) || s.Start.Equal(segmentDates[2]) {
- s.lastAccessed.Store(time.Now().Add(-time.Hour).UnixNano())
+ s.lastAccessed.Store(time.Now().Add(-2 * time.Hour).UnixNano())
s.DecRef() // Force close
}
s.DecRef() // Release our reference from segments()
(skywalking-banyandb) 09/14: fix(trace): accept out-of-order timestamps within the same traceID (#1114)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 468cb5d44349d9d2a09bd8054048ba4c6b6c891e
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Thu May 7 07:42:27 2026 +0800
fix(trace): accept out-of-order timestamps within the same traceID (#1114)
---
CHANGES.md | 1 +
banyand/trace/block_writer.go | 36 ++-------------------------
banyand/trace/block_writer_test.go | 50 ++++++++++++++++++++++++++++++++++++++
3 files changed, 53 insertions(+), 34 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 6356ba45e..c4b2c5cb7 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -31,6 +31,7 @@ Release Notes.
- Fix flaky `TestCollectWithPartialClosedSegments` by raising `SegmentIdleTimeout` so wall-clock variance on slow CI does not mark still-open segments as idle.
- Fix FODC lifecycle cache poisoning where transient `InspectAll` failures were cached for 10 minutes and masked liaison recovery; raise FODC agent and proxy timeouts from 10s to 40s.
- Fix FODC `/cluster/lifecycle` dropping zero-valued group fields (e.g. `replicas=0`, `close=false`) under `encoding/json` + `omitempty`; switch to `protojson` so all fields are emitted (nil nested messages serialize as `null`).
+- Fix trace `block_writer` panic on out-of-order timestamps within the same traceID, which dropped one trace-write batch per panic in multi-agent SkyWalking deployments. Spans of a single trace originate from independently-clocked services, and trace storage is organized by traceID rather than timestamp, so per-traceID timestamp monotonicity is not a writer invariant.
### Chores
diff --git a/banyand/trace/block_writer.go b/banyand/trace/block_writer.go
index c4267dd58..c14ec5d1e 100644
--- a/banyand/trace/block_writer.go
+++ b/banyand/trace/block_writer.go
@@ -149,11 +149,8 @@ type blockWriter struct {
primaryBlockMetadata primaryBlockMetadata
totalUncompressedSpanSizeBytes uint64
totalCount uint64
- minTimestamp int64
totalMinTimestamp int64
totalMaxTimestamp int64
- minTimestampLast int64
- maxTimestamp int64
totalBlocksCount uint64
hasWrittenBlocks bool
}
@@ -169,9 +166,6 @@ func (bw *blockWriter) reset() {
} else {
bw.tagType.reset()
}
- bw.minTimestampLast = 0
- bw.minTimestamp = 0
- bw.maxTimestamp = 0
bw.totalUncompressedSpanSizeBytes = 0
bw.totalCount = 0
bw.totalBlocksCount = 0
@@ -230,12 +224,10 @@ func (bw *blockWriter) mustWriteBlock(tid string, b *block) {
if tid < bw.tidLast {
logger.Panicf("the tid=%s cannot be smaller than the previously written tid=%s", tid, bw.tidLast)
}
- hasWrittenBlocks := bw.hasWrittenBlocks
- if !hasWrittenBlocks {
+ if !bw.hasWrittenBlocks {
bw.tidFirst = tid
bw.hasWrittenBlocks = true
}
- isSeenTid := tid == bw.tidLast
bw.tidLast = tid
bm := generateBlockMetadata()
@@ -251,16 +243,6 @@ func (bw *blockWriter) mustWriteBlock(tid string, b *block) {
if bw.totalCount == 0 || tm.max > bw.totalMaxTimestamp {
bw.totalMaxTimestamp = tm.max
}
- if !hasWrittenBlocks || tm.min < bw.minTimestamp {
- bw.minTimestamp = tm.min
- }
- if !hasWrittenBlocks || tm.max > bw.maxTimestamp {
- bw.maxTimestamp = tm.max
- }
- if isSeenTid && tm.min < bw.minTimestampLast {
- logger.Panicf("the block for tid=%s cannot contain timestamp smaller than %d, but it contains timestamp %d", tid, bw.minTimestampLast, tm.min)
- }
- bw.minTimestampLast = tm.min
bw.totalUncompressedSpanSizeBytes += bm.uncompressedSpanSizeBytes
bw.totalCount += bm.count
@@ -280,8 +262,6 @@ func (bw *blockWriter) mustFlushPrimaryBlock(data []byte) {
bw.metaData = bw.primaryBlockMetadata.marshal(bw.metaData)
}
bw.hasWrittenBlocks = false
- bw.minTimestamp = 0
- bw.maxTimestamp = 0
bw.tidFirst = ""
}
@@ -293,12 +273,10 @@ func (bw *blockWriter) mustWriteRawBlock(r *rawBlock) {
if bm.traceID < bw.tidLast {
logger.Panicf("the tid=%s cannot be smaller than the previously written tid=%s", bm.traceID, bw.tidLast)
}
- hasWrittenBlocks := bw.hasWrittenBlocks
- if !hasWrittenBlocks {
+ if !bw.hasWrittenBlocks {
bw.tidFirst = bm.traceID
bw.hasWrittenBlocks = true
}
- isSeenTid := bm.traceID == bw.tidLast
bw.tidLast = bm.traceID
if bw.traceIDFilter != nil && bw.traceIDFilter.filter != nil {
bw.traceIDFilter.filter.Add(convert.StringToBytes(bm.traceID))
@@ -312,16 +290,6 @@ func (bw *blockWriter) mustWriteRawBlock(r *rawBlock) {
if bw.totalCount == 0 || tm.max > bw.totalMaxTimestamp {
bw.totalMaxTimestamp = tm.max
}
- if !hasWrittenBlocks || tm.min < bw.minTimestamp {
- bw.minTimestamp = tm.min
- }
- if !hasWrittenBlocks || tm.max > bw.maxTimestamp {
- bw.maxTimestamp = tm.max
- }
- if isSeenTid && tm.min < bw.minTimestampLast {
- logger.Panicf("the block for tid=%s cannot contain timestamp smaller than %d, but it contains timestamp %d", bm.traceID, bw.minTimestampLast, tm.min)
- }
- bw.minTimestampLast = tm.min
bw.totalUncompressedSpanSizeBytes += bm.uncompressedSpanSizeBytes
bw.totalCount += bm.count
diff --git a/banyand/trace/block_writer_test.go b/banyand/trace/block_writer_test.go
index 754b6ec79..053de756d 100644
--- a/banyand/trace/block_writer_test.go
+++ b/banyand/trace/block_writer_test.go
@@ -165,6 +165,56 @@ func TestMultiplePrimaryBlockFlushWithPartIter(t *testing.T) {
}
}
+// TestBlockWriter_OutOfOrderTimestampsSameTraceID exercises the scenario from
+// https://github.com/apache/skywalking/issues/13860: SkyWalking OAP forwards
+// segments produced by multiple agents whose wall clocks drift relative to
+// each other, so a later batch for the same traceID can carry timestamps
+// earlier than an earlier batch. Spans of one trace have no meaningful order
+// here — the writer must accept them without panicking.
+func TestBlockWriter_OutOfOrderTimestampsSameTraceID(t *testing.T) {
+ const tid = "6359c73a002c425785500f958cdc4007.661.17779941290647127"
+
+ mp := &memPart{}
+ mp.reset()
+ bw := generateBlockWriter()
+ bw.MustInitForMemPart(mp, 1)
+
+ tagsFor := func(i int) []*tagValue {
+ return []*tagValue{
+ {tag: "service", valueType: pbv1.ValueTypeStr, value: []byte(fmt.Sprintf("svc-%d", i))},
+ }
+ }
+ mustWrite := func(timestamps []int64) {
+ spans := make([][]byte, len(timestamps))
+ spanIDs := make([]string, len(timestamps))
+ tagSets := make([][]*tagValue, len(timestamps))
+ for i, ts := range timestamps {
+ spans[i] = []byte(fmt.Sprintf("span-%d", ts))
+ spanIDs[i] = fmt.Sprintf("span-%d", ts)
+ tagSets[i] = tagsFor(i)
+ }
+ bw.MustWriteTrace(tid, spans, tagSets, timestamps, spanIDs)
+ }
+
+ // First batch: minTS=1_777_994_129_833_000_000.
+ mustWrite([]int64{1_777_994_129_833_000_000, 1_777_994_129_900_000_000})
+ // Second batch for the same tid carries an earlier minTS — this used to
+ // fail the per-traceID timestamp monotonicity check and discard the batch
+ // on production traffic.
+ require.NotPanics(t, func() {
+ mustWrite([]int64{1_777_994_129_064_000_000})
+ })
+
+ bw.Flush(&mp.partMetadata, &mp.traceIDFilter, &mp.tagType)
+ releaseBlockWriter(bw)
+
+ assert.Equal(t, uint64(3), mp.partMetadata.TotalCount, "all 3 spans must be persisted")
+ assert.Equal(t, uint64(2), mp.partMetadata.BlocksCount, "two MustWriteTrace calls produce two blocks")
+ assert.Equal(t, int64(1_777_994_129_064_000_000), mp.partMetadata.MinTimestamp,
+ "part-level MinTimestamp aggregates across batches regardless of arrival order")
+ assert.Equal(t, int64(1_777_994_129_900_000_000), mp.partMetadata.MaxTimestamp)
+}
+
func generateLargeTraceSet() *traces {
// maxUncompressedPrimaryBlockSize = 128KB
// Block metadata contains: traceID, timestamps, count, uncompressed size, and tag metadata
(skywalking-banyandb) 01/14: Stable Segment Endtime (#1051)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 7f7a5683ba6964857df3c646ee42715921e2897b
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed Apr 8 18:06:01 2026 +0800
Stable Segment Endtime (#1051)
---
.gitignore | 3 +
CHANGES.md | 13 ++
banyand/internal/storage/segment.go | 38 ++++--
banyand/internal/storage/segment_test.go | 219 +++++++++++++++++++++++++++++++
banyand/internal/storage/version.go | 23 +++-
banyand/internal/storage/version_test.go | 63 +++++++++
banyand/internal/storage/versions.yml | 1 +
7 files changed, 348 insertions(+), 12 deletions(-)
diff --git a/.gitignore b/.gitignore
index 9af8ace51..3235cb1ac 100644
--- a/.gitignore
+++ b/.gitignore
@@ -81,6 +81,9 @@ gomock_reflect*
# eBPF generated files and binaries
fodc/agent/internal/ktm/iomonitor/ebpf/generated/vmlinux.h
+<<<<<<< HEAD
# OpenSSL serial files
*.srl
+=======
+>>>>>>> 29e87c9d (Stable Segment Endtime (#1051))
diff --git a/CHANGES.md b/CHANGES.md
index 80c4f6c93..ace8b3034 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -2,7 +2,20 @@
Release Notes.
+<<<<<<< HEAD
## 0.10.2
+=======
+## 0.11.0
+
+### Features
+
+- Organize access logs under a dedicated "accesslog" subdirectory to improve log organization and separation from other application data.
+- Collect BanyanDB data on e2e test failure for CI debugging.
+- Add log query e2e test.
+- Sync lifecycle e2e test from SkyWalking stages test.
+- Add periodic health check for property schema connection.
+- Persist segment end time in per-segment metadata so boundaries don't shift across restarts or config changes.
+>>>>>>> 29e87c9d (Stable Segment Endtime (#1051))
### Bug Fixes
diff --git a/banyand/internal/storage/segment.go b/banyand/internal/storage/segment.go
index e4239e0bc..381a11478 100644
--- a/banyand/internal/storage/segment.go
+++ b/banyand/internal/storage/segment.go
@@ -19,6 +19,7 @@ package storage
import (
"context"
+ "encoding/json"
"fmt"
"io/fs"
"path"
@@ -33,7 +34,6 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
- "github.com/apache/skywalking-banyandb/pkg/convert"
banyanfs "github.com/apache/skywalking-banyandb/pkg/fs"
"github.com/apache/skywalking-banyandb/pkg/index/inverted"
"github.com/apache/skywalking-banyandb/pkg/logger"
@@ -491,23 +491,32 @@ func (sc *segmentController[T, O]) open() error {
return nil
}
metadataPath := path.Join(segmentPath, metadataFilename)
- version, err := sc.lfs.Read(metadataPath)
- if err != nil {
- if errors.Is(err, fs.ErrNotExist) {
+ rawMeta, readErr := sc.lfs.Read(metadataPath)
+ if readErr != nil {
+ if errors.Is(readErr, fs.ErrNotExist) {
emptySegments = append(emptySegments, segmentPath)
return nil
}
- return err
+ return readErr
}
- if len(version) == 0 {
+ if len(rawMeta) == 0 {
emptySegments = append(emptySegments, segmentPath)
return nil
}
- if err = checkVersion(convert.BytesToString(version)); err != nil {
- return err
+ meta, parseErr := readSegmentMeta(rawMeta)
+ if parseErr != nil {
+ return parseErr
}
- _, err = sc.load(start, end, sc.location)
- return err
+ segmentEnd := end
+ if meta.EndTime != "" {
+ parsedEnd, timeErr := time.Parse(time.RFC3339Nano, meta.EndTime)
+ if timeErr != nil {
+ return timeErr
+ }
+ segmentEnd = parsedEnd
+ }
+ _, loadErr := sc.load(start, segmentEnd, sc.location)
+ return loadErr
})
if len(emptySegments) > 0 {
sc.l.Warn().Strs("segments", emptySegments).Msg("empty segments found, removing them.")
@@ -551,7 +560,14 @@ func (sc *segmentController[T, O]) create(start time.Time) (*segment[T, O], erro
}
segPath := path.Join(sc.location, fmt.Sprintf(segTemplate, sc.format(start)))
sc.lfs.MkdirPanicIfExist(segPath, DirPerm)
- data := []byte(currentVersion)
+ meta := segmentMeta{
+ Version: currentVersion,
+ EndTime: end.Format(time.RFC3339Nano),
+ }
+ data, marshalErr := json.Marshal(meta)
+ if marshalErr != nil {
+ logger.Panicf("cannot marshal segment metadata: %s", marshalErr)
+ }
metadataPath := filepath.Join(segPath, metadataFilename)
lf, err := sc.lfs.CreateLockFile(metadataPath, FilePerm)
if err != nil {
diff --git a/banyand/internal/storage/segment_test.go b/banyand/internal/storage/segment_test.go
index 7bcc1063b..9a55dd358 100644
--- a/banyand/internal/storage/segment_test.go
+++ b/banyand/internal/storage/segment_test.go
@@ -19,6 +19,7 @@ package storage
import (
"context"
+ "encoding/json"
"fmt"
"os"
"path/filepath"
@@ -641,3 +642,221 @@ func TestDeleteExpiredSegmentsWithClosedSegments(t *testing.T) {
"Remaining segment %d should be from the expected date", i)
}
}
+
+func TestCreateSegmentWritesJSONMetadata(t *testing.T) {
+ tempDir, cleanup := setupTestEnvironment(t)
+ defer cleanup()
+
+ ctx := context.Background()
+ l := logger.GetLogger("test-segment-metadata")
+ ctx = context.WithValue(ctx, logger.ContextKey, l)
+ ctx = common.SetPosition(ctx, func(_ common.Position) common.Position {
+ return common.Position{
+ Database: "test-db",
+ Stage: "test-stage",
+ }
+ })
+
+ opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
+ _ timestamp.TimeRange, _ mockTSTableOpener, _ any,
+ ) (mockTSTable, error) {
+ return mockTSTable{ID: common.ShardID(0)}, nil
+ },
+ ShardNum: 1,
+ SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
+ TTL: IntervalRule{Unit: DAY, Num: 7},
+ SeriesIndexFlushTimeoutSeconds: 10,
+ SeriesIndexCacheMaxBytes: 1024 * 1024,
+ }
+
+ serviceCache := NewServiceCache().(*serviceCache)
+ sc := newSegmentController[mockTSTable, mockTSTableOpener](
+ ctx,
+ tempDir,
+ l,
+ opts,
+ nil,
+ nil,
+ 5*time.Minute,
+ fs.NewLocalFileSystemWithLoggerAndLimit(logger.GetLogger("storage"), opts.MemoryLimit),
+ serviceCache,
+ group,
+ )
+
+ now := time.Now().UTC()
+ startTime := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
+
+ seg, createErr := sc.create(startTime)
+ require.NoError(t, createErr)
+ require.NotNil(t, seg)
+
+ // Read metadata from disk and verify it's JSON with endTime
+ suffix := startTime.Format(dayFormat)
+ metadataPath := filepath.Join(tempDir, fmt.Sprintf("seg-%s", suffix), metadataFilename)
+ rawMeta, readErr := os.ReadFile(metadataPath)
+ require.NoError(t, readErr)
+
+ meta, parseErr := readSegmentMeta(rawMeta)
+ require.NoError(t, parseErr)
+ assert.Equal(t, currentVersion, meta.Version)
+ assert.NotEmpty(t, meta.EndTime, "endTime should be persisted in metadata")
+
+ // Verify the endTime matches the segment's End
+ expectedEnd := startTime.Add(24 * time.Hour)
+ assert.Equal(t, expectedEnd.Format(time.RFC3339Nano), meta.EndTime)
+
+ seg.DecRef()
+}
+
+func TestOpenReadsPersistedEndTime(t *testing.T) {
+ tempDir, cleanup := setupTestEnvironment(t)
+ defer cleanup()
+
+ ctx := context.Background()
+ l := logger.GetLogger("test-open-endtime")
+ ctx = context.WithValue(ctx, logger.ContextKey, l)
+ ctx = common.SetPosition(ctx, func(_ common.Position) common.Position {
+ return common.Position{
+ Database: "test-db",
+ Stage: "test-stage",
+ }
+ })
+
+ group := "test-group"
+ opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
+ _ timestamp.TimeRange, _ mockTSTableOpener, _ any,
+ ) (mockTSTable, error) {
+ return mockTSTable{ID: common.ShardID(0)}, nil
+ },
+ ShardNum: 1,
+ SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
+ TTL: IntervalRule{Unit: DAY, Num: 30},
+ SeriesIndexFlushTimeoutSeconds: 10,
+ SeriesIndexCacheMaxBytes: 1024 * 1024,
+ }
+
+ serviceCache := NewServiceCache().(*serviceCache)
+ sc := newSegmentController[mockTSTable, mockTSTableOpener](
+ ctx,
+ tempDir,
+ l,
+ opts,
+ nil,
+ nil,
+ 5*time.Minute,
+ fs.NewLocalFileSystemWithLoggerAndLimit(logger.GetLogger("storage"), opts.MemoryLimit),
+ serviceCache,
+ group,
+ )
+
+ now := time.Now().UTC()
+ day1 := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
+ day2 := day1.Add(24 * time.Hour)
+
+ // Manually create segment directories with JSON metadata.
+ // day1 with a specific endTime that differs from the default NextTime.
+ customEnd := day1.Add(12 * time.Hour)
+ segPath1 := filepath.Join(tempDir, "seg-"+day1.Format(dayFormat))
+ require.NoError(t, os.MkdirAll(segPath1, DirPerm))
+ meta1 := segmentMeta{Version: currentVersion, EndTime: customEnd.Format(time.RFC3339Nano)}
+ meta1Data, marshalErr := json.Marshal(meta1)
+ require.NoError(t, marshalErr)
+ require.NoError(t, os.WriteFile(filepath.Join(segPath1, metadataFilename), meta1Data, FilePerm))
+
+ // day2 with standard endTime.
+ segPath2 := filepath.Join(tempDir, "seg-"+day2.Format(dayFormat))
+ require.NoError(t, os.MkdirAll(segPath2, DirPerm))
+ meta2 := segmentMeta{Version: currentVersion, EndTime: day2.Add(24 * time.Hour).Format(time.RFC3339Nano)}
+ meta2Data, marshalErr := json.Marshal(meta2)
+ require.NoError(t, marshalErr)
+ require.NoError(t, os.WriteFile(filepath.Join(segPath2, metadataFilename), meta2Data, FilePerm))
+
+ // Open the controller (loads segments from disk).
+ openErr := sc.open()
+ require.NoError(t, openErr)
+
+ // Verify segments loaded correctly.
+ require.Len(t, sc.lst, 2)
+
+ // First segment should have the custom endTime from metadata.
+ assert.Equal(t, customEnd, sc.lst[0].End, "segment 0 should use persisted endTime")
+ // Second segment should have its endTime from metadata.
+ assert.Equal(t, day2.Add(24*time.Hour), sc.lst[1].End, "segment 1 should use persisted endTime")
+
+ for _, seg := range sc.lst {
+ seg.DecRef()
+ }
+}
+
+func TestOpenFallbackOldFormatMetadata(t *testing.T) {
+ tempDir, cleanup := setupTestEnvironment(t)
+ defer cleanup()
+
+ ctx := context.Background()
+ l := logger.GetLogger("test-open-fallback")
+ ctx = context.WithValue(ctx, logger.ContextKey, l)
+ ctx = common.SetPosition(ctx, func(_ common.Position) common.Position {
+ return common.Position{
+ Database: "test-db",
+ Stage: "test-stage",
+ }
+ })
+
+ group := "test-group"
+ opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
+ _ timestamp.TimeRange, _ mockTSTableOpener, _ any,
+ ) (mockTSTable, error) {
+ return mockTSTable{ID: common.ShardID(0)}, nil
+ },
+ ShardNum: 1,
+ SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
+ TTL: IntervalRule{Unit: DAY, Num: 30},
+ SeriesIndexFlushTimeoutSeconds: 10,
+ SeriesIndexCacheMaxBytes: 1024 * 1024,
+ }
+
+ serviceCache := NewServiceCache().(*serviceCache)
+ sc := newSegmentController[mockTSTable, mockTSTableOpener](
+ ctx,
+ tempDir,
+ l,
+ opts,
+ nil,
+ nil,
+ 5*time.Minute,
+ fs.NewLocalFileSystemWithLoggerAndLimit(logger.GetLogger("storage"), opts.MemoryLimit),
+ serviceCache,
+ group,
+ )
+
+ now := time.Now()
+ day1 := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local)
+ day2 := day1.Add(24 * time.Hour)
+
+ // Create segments with OLD format metadata (plain version string).
+ segPath1 := filepath.Join(tempDir, "seg-"+day1.Format(dayFormat))
+ require.NoError(t, os.MkdirAll(segPath1, DirPerm))
+ require.NoError(t, os.WriteFile(filepath.Join(segPath1, metadataFilename), []byte("1.4.0"), FilePerm))
+
+ segPath2 := filepath.Join(tempDir, "seg-"+day2.Format(dayFormat))
+ require.NoError(t, os.MkdirAll(segPath2, DirPerm))
+ require.NoError(t, os.WriteFile(filepath.Join(segPath2, metadataFilename), []byte("1.4.0"), FilePerm))
+
+ // Open should succeed with fallback end time computation.
+ openErr := sc.open()
+ require.NoError(t, openErr)
+
+ require.Len(t, sc.lst, 2)
+
+ // First segment's end should be day2's start (fallback: end = next segment's start).
+ assert.Equal(t, day2, sc.lst[0].End, "old format should use fallback end time")
+ // Second segment's end should be day2 + 24h (fallback: end = NextTime(start)).
+ assert.Equal(t, day2.Add(24*time.Hour), sc.lst[1].End, "last segment should use NextTime fallback")
+
+ for _, seg := range sc.lst {
+ seg.DecRef()
+ }
+}
diff --git a/banyand/internal/storage/version.go b/banyand/internal/storage/version.go
index 55a92f04b..ddea79a33 100644
--- a/banyand/internal/storage/version.go
+++ b/banyand/internal/storage/version.go
@@ -28,7 +28,7 @@ import (
const (
metadataFilename = "metadata"
- currentVersion = "1.4.0"
+ currentVersion = "1.5.0"
compatibleVersionsKey = "versions"
compatibleVersionsFilename = "versions.yml"
)
@@ -49,6 +49,27 @@ func checkVersion(version string) error {
return errors.WithMessagef(errVersionIncompatible, "incompatible version %s, supported versions: %s", version, strings.Join(compatibleVersions, ", "))
}
+type segmentMeta struct {
+ Version string `json:"version"`
+ EndTime string `json:"endTime,omitempty"`
+}
+
+func readSegmentMeta(data []byte) (segmentMeta, error) {
+ var meta segmentMeta
+ trimmed := strings.TrimSpace(string(data))
+ if len(trimmed) > 0 && trimmed[0] == '{' {
+ if unmarshalErr := json.Unmarshal(data, &meta); unmarshalErr != nil {
+ return segmentMeta{}, unmarshalErr
+ }
+ } else {
+ meta.Version = trimmed
+ }
+ if checkErr := checkVersion(meta.Version); checkErr != nil {
+ return segmentMeta{}, checkErr
+ }
+ return meta, nil
+}
+
func readCompatibleVersions() []string {
i, err := versionFS.ReadFile(compatibleVersionsFilename)
if err != nil {
diff --git a/banyand/internal/storage/version_test.go b/banyand/internal/storage/version_test.go
new file mode 100644
index 000000000..726e6d8a2
--- /dev/null
+++ b/banyand/internal/storage/version_test.go
@@ -0,0 +1,63 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package storage
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestReadSegmentMeta_NewFormat(t *testing.T) {
+ data := []byte(`{"version":"1.4.0","endTime":"2026-04-07T00:00:00+08:00"}`)
+ meta, err := readSegmentMeta(data)
+ require.NoError(t, err)
+ assert.Equal(t, "1.4.0", meta.Version)
+ assert.Equal(t, "2026-04-07T00:00:00+08:00", meta.EndTime)
+}
+
+func TestReadSegmentMeta_OldFormat(t *testing.T) {
+ data := []byte("1.4.0")
+ meta, err := readSegmentMeta(data)
+ require.NoError(t, err)
+ assert.Equal(t, "1.4.0", meta.Version)
+ assert.Equal(t, "", meta.EndTime)
+}
+
+func TestReadSegmentMeta_OldFormatWithNewline(t *testing.T) {
+ data := []byte("1.4.0\n")
+ meta, err := readSegmentMeta(data)
+ require.NoError(t, err)
+ assert.Equal(t, "1.4.0", meta.Version)
+ assert.Equal(t, "", meta.EndTime)
+}
+
+func TestReadSegmentMeta_IncompatibleVersion(t *testing.T) {
+ data := []byte(`{"version":"0.1.0","endTime":"2026-04-07T00:00:00+08:00"}`)
+ _, err := readSegmentMeta(data)
+ assert.Error(t, err)
+}
+
+func TestReadSegmentMeta_NewFormatNoEndTime(t *testing.T) {
+ data := []byte(`{"version":"1.4.0"}`)
+ meta, err := readSegmentMeta(data)
+ require.NoError(t, err)
+ assert.Equal(t, "1.4.0", meta.Version)
+ assert.Equal(t, "", meta.EndTime)
+}
diff --git a/banyand/internal/storage/versions.yml b/banyand/internal/storage/versions.yml
index e5cb937c5..ddac13c49 100644
--- a/banyand/internal/storage/versions.yml
+++ b/banyand/internal/storage/versions.yml
@@ -15,3 +15,4 @@
versions:
- 1.4.0
+- 1.5.0
(skywalking-banyandb) 03/14: fix(ui): correct query editor refresh/reset behavior and BydbQL keyword highlighting (#1045)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 7ff0d58285b1908da3bb66b6d8f7b99bc24d4f98
Author: Fine0830 <fa...@gmail.com>
AuthorDate: Mon Apr 6 20:10:34 2026 +0800
fix(ui): correct query editor refresh/reset behavior and BydbQL keyword highlighting (#1045)
---
CHANGES.md | 1 +
dist/LICENSE | 4 +-
ui/LICENSE | 4 +-
ui/package-lock.json | 417 ++++++++++++++--------------
ui/src/components/CodeMirror/bydbql-mode.js | 9 +
ui/src/components/Property/PropertyRead.vue | 36 ++-
ui/src/components/Read/index.vue | 14 +-
7 files changed, 258 insertions(+), 227 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index ecbffb7ee..26090e881 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -12,6 +12,7 @@ Release Notes.
- MCP: Add validation for properties and harden the mcp server.
- Fix property schema client connection not stable after data node restarted.
- Fix flaky on-disk integration tests caused by Ginkgo v2 random container shuffling closing gRPC connections prematurely.
+- ui: fix query editor refresh/reset behavior and BydbQL keyword highlighting.
## 0.10.0
diff --git a/dist/LICENSE b/dist/LICENSE
index 45ae5033b..bf915f4c3 100644
--- a/dist/LICENSE
+++ b/dist/LICENSE
@@ -563,8 +563,8 @@ MIT licenses
js-yaml 4.1.1 MIT
jsonfile 6.2.1 MIT
lie 3.3.0 MIT
- lodash 4.17.23 MIT
- lodash-es 4.17.23 MIT
+ lodash 4.18.1 MIT
+ lodash-es 4.18.1 MIT
lodash-unified 1.0.3 MIT
magic-string 0.30.21 MIT
memoize-one 6.0.0 MIT
diff --git a/ui/LICENSE b/ui/LICENSE
index b0d5014f6..8ffffbc7a 100644
--- a/ui/LICENSE
+++ b/ui/LICENSE
@@ -108,8 +108,8 @@ MIT licenses
js-yaml 4.1.1 MIT
jsonfile 6.2.1 MIT
lie 3.3.0 MIT
- lodash 4.17.23 MIT
- lodash-es 4.17.23 MIT
+ lodash 4.18.1 MIT
+ lodash-es 4.18.1 MIT
lodash-unified 1.0.3 MIT
magic-string 0.30.21 MIT
memoize-one 6.0.0 MIT
diff --git a/ui/package-lock.json b/ui/package-lock.json
index d007068c3..529e09c21 100644
--- a/ui/package-lock.json
+++ b/ui/package-lock.json
@@ -951,9 +951,9 @@
}
},
"node_modules/@rollup/rollup-android-arm-eabi": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.2.tgz",
- "integrity": "sha512-dnlp69efPPg6Uaw2dVqzWRfAWRnYVb1XJ8CyyhIbZeaq4CA5/mLeZ1IEt9QqQxmbdvagjLIm2ZL8BxXv5lH4Yw==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.0.tgz",
+ "integrity": "sha512-WOhNW9K8bR3kf4zLxbfg6Pxu2ybOUbB2AjMDHSQx86LIF4rH4Ft7vmMwNt0loO0eonglSNy4cpD3MKXXKQu0/A==",
"cpu": [
"arm"
],
@@ -965,9 +965,9 @@
]
},
"node_modules/@rollup/rollup-android-arm64": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.2.tgz",
- "integrity": "sha512-OqZTwDRDchGRHHm/hwLOL7uVPB9aUvI0am/eQuWMNyFHf5PSEQmyEeYYheA0EPPKUO/l0uigCp+iaTjoLjVoHg==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.0.tgz",
+ "integrity": "sha512-u6JHLll5QKRvjciE78bQXDmqRqNs5M/3GVqZeMwvmjaNODJih/WIrJlFVEihvV0MiYFmd+ZyPr9wxOVbPAG2Iw==",
"cpu": [
"arm64"
],
@@ -979,9 +979,9 @@
]
},
"node_modules/@rollup/rollup-darwin-arm64": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.2.tgz",
- "integrity": "sha512-UwRE7CGpvSVEQS8gUMBe1uADWjNnVgP3Iusyda1nSRwNDCsRjnGc7w6El6WLQsXmZTbLZx9cecegumcitNfpmA==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.0.tgz",
+ "integrity": "sha512-qEF7CsKKzSRc20Ciu2Zw1wRrBz4g56F7r/vRwY430UPp/nt1x21Q/fpJ9N5l47WWvJlkNCPJz3QRVw008fi7yA==",
"cpu": [
"arm64"
],
@@ -993,9 +993,9 @@
]
},
"node_modules/@rollup/rollup-darwin-x64": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.2.tgz",
- "integrity": "sha512-gjEtURKLCC5VXm1I+2i1u9OhxFsKAQJKTVB8WvDAHF+oZlq0GTVFOlTlO1q3AlCTE/DF32c16ESvfgqR7343/g==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.0.tgz",
+ "integrity": "sha512-WADYozJ4QCnXCH4wPB+3FuGmDPoFseVCUrANmA5LWwGmC6FL14BWC7pcq+FstOZv3baGX65tZ378uT6WG8ynTw==",
"cpu": [
"x64"
],
@@ -1007,9 +1007,9 @@
]
},
"node_modules/@rollup/rollup-freebsd-arm64": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.2.tgz",
- "integrity": "sha512-Bcl6CYDeAgE70cqZaMojOi/eK63h5Me97ZqAQoh77VPjMysA/4ORQBRGo3rRy45x4MzVlU9uZxs8Uwy7ZaKnBw==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.0.tgz",
+ "integrity": "sha512-6b8wGHJlDrGeSE3aH5mGNHBjA0TTkxdoNHik5EkvPHCt351XnigA4pS7Wsj/Eo9Y8RBU6f35cjN9SYmCFBtzxw==",
"cpu": [
"arm64"
],
@@ -1021,9 +1021,9 @@
]
},
"node_modules/@rollup/rollup-freebsd-x64": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.2.tgz",
- "integrity": "sha512-LU+TPda3mAE2QB0/Hp5VyeKJivpC6+tlOXd1VMoXV/YFMvk/MNk5iXeBfB4MQGRWyOYVJ01625vjkr0Az98OJQ==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.0.tgz",
+ "integrity": "sha512-h25Ga0t4jaylMB8M/JKAyrvvfxGRjnPQIR8lnCayyzEjEOx2EJIlIiMbhpWxDRKGKF8jbNH01NnN663dH638mA==",
"cpu": [
"x64"
],
@@ -1035,9 +1035,9 @@
]
},
"node_modules/@rollup/rollup-linux-arm-gnueabihf": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.2.tgz",
- "integrity": "sha512-2QxQrM+KQ7DAW4o22j+XZ6RKdxjLD7BOWTP0Bv0tmjdyhXSsr2Ul1oJDQqh9Zf5qOwTuTc7Ek83mOFaKnodPjg==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.0.tgz",
+ "integrity": "sha512-RzeBwv0B3qtVBWtcuABtSuCzToo2IEAIQrcyB/b2zMvBWVbjo8bZDjACUpnaafaxhTw2W+imQbP2BD1usasK4g==",
"cpu": [
"arm"
],
@@ -1049,9 +1049,9 @@
]
},
"node_modules/@rollup/rollup-linux-arm-musleabihf": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.2.tgz",
- "integrity": "sha512-TbziEu2DVsTEOPif2mKWkMeDMLoYjx95oESa9fkQQK7r/Orta0gnkcDpzwufEcAO2BLBsD7mZkXGFqEdMRRwfw==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.0.tgz",
+ "integrity": "sha512-Sf7zusNI2CIU1HLzuu9Tc5YGAHEZs5Lu7N1ssJG4Tkw6e0MEsN7NdjUDDfGNHy2IU+ENyWT+L2obgWiguWibWQ==",
"cpu": [
"arm"
],
@@ -1063,9 +1063,9 @@
]
},
"node_modules/@rollup/rollup-linux-arm64-gnu": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.2.tgz",
- "integrity": "sha512-bO/rVDiDUuM2YfuCUwZ1t1cP+/yqjqz+Xf2VtkdppefuOFS2OSeAfgafaHNkFn0t02hEyXngZkxtGqXcXwO8Rg==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.0.tgz",
+ "integrity": "sha512-DX2x7CMcrJzsE91q7/O02IJQ5/aLkVtYFryqCjduJhUfGKG6yJV8hxaw8pZa93lLEpPTP/ohdN4wFz7yp/ry9A==",
"cpu": [
"arm64"
],
@@ -1077,9 +1077,9 @@
]
},
"node_modules/@rollup/rollup-linux-arm64-musl": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.2.tgz",
- "integrity": "sha512-hr26p7e93Rl0Za+JwW7EAnwAvKkehh12BU1Llm9Ykiibg4uIr2rbpxG9WCf56GuvidlTG9KiiQT/TXT1yAWxTA==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.0.tgz",
+ "integrity": "sha512-09EL+yFVbJZlhcQfShpswwRZ0Rg+z/CsSELFCnPt3iK+iqwGsI4zht3secj5vLEs957QvFFXnzAT0FFPIxSrkQ==",
"cpu": [
"arm64"
],
@@ -1091,9 +1091,9 @@
]
},
"node_modules/@rollup/rollup-linux-loong64-gnu": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.2.tgz",
- "integrity": "sha512-pOjB/uSIyDt+ow3k/RcLvUAOGpysT2phDn7TTUB3n75SlIgZzM6NKAqlErPhoFU+npgY3/n+2HYIQVbF70P9/A==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.0.tgz",
+ "integrity": "sha512-i9IcCMPr3EXm8EQg5jnja0Zyc1iFxJjZWlb4wr7U2Wx/GrddOuEafxRdMPRYVaXjgbhvqalp6np07hN1w9kAKw==",
"cpu": [
"loong64"
],
@@ -1105,9 +1105,9 @@
]
},
"node_modules/@rollup/rollup-linux-loong64-musl": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.2.tgz",
- "integrity": "sha512-2/w+q8jszv9Ww1c+6uJT3OwqhdmGP2/4T17cu8WuwyUuuaCDDJ2ojdyYwZzCxx0GcsZBhzi3HmH+J5pZNXnd+Q==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.0.tgz",
+ "integrity": "sha512-DGzdJK9kyJ+B78MCkWeGnpXJ91tK/iKA6HwHxF4TAlPIY7GXEvMe8hBFRgdrR9Ly4qebR/7gfUs9y2IoaVEyog==",
"cpu": [
"loong64"
],
@@ -1119,9 +1119,9 @@
]
},
"node_modules/@rollup/rollup-linux-ppc64-gnu": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.2.tgz",
- "integrity": "sha512-11+aL5vKheYgczxtPVVRhdptAM2H7fcDR5Gw4/bTcteuZBlH4oP9f5s9zYO9aGZvoGeBpqXI/9TZZihZ609wKw==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.0.tgz",
+ "integrity": "sha512-RwpnLsqC8qbS8z1H1AxBA1H6qknR4YpPR9w2XX0vo2Sz10miu57PkNcnHVaZkbqyw/kUWfKMI73jhmfi9BRMUQ==",
"cpu": [
"ppc64"
],
@@ -1133,9 +1133,9 @@
]
},
"node_modules/@rollup/rollup-linux-ppc64-musl": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.2.tgz",
- "integrity": "sha512-i16fokAGK46IVZuV8LIIwMdtqhin9hfYkCh8pf8iC3QU3LpwL+1FSFGej+O7l3E/AoknL6Dclh2oTdnRMpTzFQ==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.0.tgz",
+ "integrity": "sha512-Z8pPf54Ly3aqtdWC3G4rFigZgNvd+qJlOE52fmko3KST9SoGfAdSRCwyoyG05q1HrrAblLbk1/PSIV+80/pxLg==",
"cpu": [
"ppc64"
],
@@ -1147,9 +1147,9 @@
]
},
"node_modules/@rollup/rollup-linux-riscv64-gnu": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.2.tgz",
- "integrity": "sha512-49FkKS6RGQoriDSK/6E2GkAsAuU5kETFCh7pG4yD/ylj9rKhTmO3elsnmBvRD4PgJPds5W2PkhC82aVwmUcJ7A==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.0.tgz",
+ "integrity": "sha512-3a3qQustp3COCGvnP4SvrMHnPQ9d1vzCakQVRTliaz8cIp/wULGjiGpbcqrkv0WrHTEp8bQD/B3HBjzujVWLOA==",
"cpu": [
"riscv64"
],
@@ -1161,9 +1161,9 @@
]
},
"node_modules/@rollup/rollup-linux-riscv64-musl": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.2.tgz",
- "integrity": "sha512-mjYNkHPfGpUR00DuM1ZZIgs64Hpf4bWcz9Z41+4Q+pgDx73UwWdAYyf6EG/lRFldmdHHzgrYyge5akFUW0D3mQ==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.0.tgz",
+ "integrity": "sha512-pjZDsVH/1VsghMJ2/kAaxt6dL0psT6ZexQVrijczOf+PeP2BUqTHYejk3l6TlPRydggINOeNRhvpLa0AYpCWSQ==",
"cpu": [
"riscv64"
],
@@ -1175,9 +1175,9 @@
]
},
"node_modules/@rollup/rollup-linux-s390x-gnu": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.2.tgz",
- "integrity": "sha512-ALyvJz965BQk8E9Al/JDKKDLH2kfKFLTGMlgkAbbYtZuJt9LU8DW3ZoDMCtQpXAltZxwBHevXz5u+gf0yA0YoA==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.0.tgz",
+ "integrity": "sha512-3ObQs0BhvPgiUVZrN7gqCSvmFuMWvWvsjG5ayJ3Lraqv+2KhOsp+pUbigqbeWqueGIsnn+09HBw27rJ+gYK4VQ==",
"cpu": [
"s390x"
],
@@ -1189,9 +1189,9 @@
]
},
"node_modules/@rollup/rollup-linux-x64-gnu": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.2.tgz",
- "integrity": "sha512-UQjrkIdWrKI626Du8lCQ6MJp/6V1LAo2bOK9OTu4mSn8GGXIkPXk/Vsp4bLHCd9Z9Iz2OTEaokUE90VweJgIYQ==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.0.tgz",
+ "integrity": "sha512-EtylprDtQPdS5rXvAayrNDYoJhIz1/vzN2fEubo3yLE7tfAw+948dO0g4M0vkTVFhKojnF+n6C8bDNe+gDRdTg==",
"cpu": [
"x64"
],
@@ -1203,9 +1203,9 @@
]
},
"node_modules/@rollup/rollup-linux-x64-musl": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.2.tgz",
- "integrity": "sha512-bTsRGj6VlSdn/XD4CGyzMnzaBs9bsRxy79eTqTCBsA8TMIEky7qg48aPkvJvFe1HyzQ5oMZdg7AnVlWQSKLTnw==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.0.tgz",
+ "integrity": "sha512-k09oiRCi/bHU9UVFqD17r3eJR9bn03TyKraCrlz5ULFJGdJGi7VOmm9jl44vOJvRJ6P7WuBi/s2A97LxxHGIdw==",
"cpu": [
"x64"
],
@@ -1217,9 +1217,9 @@
]
},
"node_modules/@rollup/rollup-openbsd-x64": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.2.tgz",
- "integrity": "sha512-6d4Z3534xitaA1FcMWP7mQPq5zGwBmGbhphh2DwaA1aNIXUu3KTOfwrWpbwI4/Gr0uANo7NTtaykFyO2hPuFLg==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.0.tgz",
+ "integrity": "sha512-1o/0/pIhozoSaDJoDcec+IVLbnRtQmHwPV730+AOD29lHEEo4F5BEUB24H0OBdhbBBDwIOSuf7vgg0Ywxdfiiw==",
"cpu": [
"x64"
],
@@ -1231,9 +1231,9 @@
]
},
"node_modules/@rollup/rollup-openharmony-arm64": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.2.tgz",
- "integrity": "sha512-NetAg5iO2uN7eB8zE5qrZ3CSil+7IJt4WDFLcC75Ymywq1VZVD6qJ6EvNLjZ3rEm6gB7XW5JdT60c6MN35Z85Q==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.0.tgz",
+ "integrity": "sha512-pESDkos/PDzYwtyzB5p/UoNU/8fJo68vcXM9ZW2V0kjYayj1KaaUfi1NmTUTUpMn4UhU4gTuK8gIaFO4UGuMbA==",
"cpu": [
"arm64"
],
@@ -1245,9 +1245,9 @@
]
},
"node_modules/@rollup/rollup-win32-arm64-msvc": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.2.tgz",
- "integrity": "sha512-NCYhOotpgWZ5kdxCZsv6Iudx0wX8980Q/oW4pNFNihpBKsDbEA1zpkfxJGC0yugsUuyDZ7gL37dbzwhR0VI7pQ==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.0.tgz",
+ "integrity": "sha512-hj1wFStD7B1YBeYmvY+lWXZ7ey73YGPcViMShYikqKT1GtstIKQAtfUI6yrzPjAy/O7pO0VLXGmUVWXQMaYgTQ==",
"cpu": [
"arm64"
],
@@ -1259,9 +1259,9 @@
]
},
"node_modules/@rollup/rollup-win32-ia32-msvc": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.2.tgz",
- "integrity": "sha512-RXsaOqXxfoUBQoOgvmmijVxJnW2IGB0eoMO7F8FAjaj0UTywUO/luSqimWBJn04WNgUkeNhh7fs7pESXajWmkg==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.0.tgz",
+ "integrity": "sha512-SyaIPFoxmUPlNDq5EHkTbiKzmSEmq/gOYFI/3HHJ8iS/v1mbugVa7dXUzcJGQfoytp9DJFLhHH4U3/eTy2Bq4w==",
"cpu": [
"ia32"
],
@@ -1273,9 +1273,9 @@
]
},
"node_modules/@rollup/rollup-win32-x64-gnu": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.2.tgz",
- "integrity": "sha512-qdAzEULD+/hzObedtmV6iBpdL5TIbKVztGiK7O3/KYSf+HIzU257+MX1EXJcyIiDbMAqmbwaufcYPvyRryeZtA==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.0.tgz",
+ "integrity": "sha512-RdcryEfzZr+lAr5kRm2ucN9aVlCCa2QNq4hXelZxb8GG0NJSazq44Z3PCCc8wISRuCVnGs0lQJVX5Vp6fKA+IA==",
"cpu": [
"x64"
],
@@ -1287,9 +1287,9 @@
]
},
"node_modules/@rollup/rollup-win32-x64-msvc": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.2.tgz",
- "integrity": "sha512-Nd/SgG27WoA9e+/TdK74KnHz852TLa94ovOYySo/yMPuTmpckK/jIF2jSwS3g7ELSKXK13/cVdmg1Z/DaCWKxA==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.0.tgz",
+ "integrity": "sha512-PrsWNQ8BuE00O3Xsx3ALh2Df8fAj9+cvvX9AIA6o4KpATR98c9mud4XtDWVvsEuyia5U4tVSTKygawyJkjm60w==",
"cpu": [
"x64"
],
@@ -1317,9 +1317,9 @@
}
},
"node_modules/@tanstack/query-core": {
- "version": "5.100.6",
- "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.100.6.tgz",
- "integrity": "sha512-Os2CPUr98to98RYm+D4qGqGkiffn7MGSyl2547a4MljVkHE30AMJRqTiyCqBfMwzAx/I91vCkAxp5tHSla6Twg==",
+ "version": "5.95.2",
+ "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.95.2.tgz",
+ "integrity": "sha512-o4T8vZHZET4Bib3jZ/tCW9/7080urD4c+0/AUaYVpIqOsr7y0reBc1oX3ttNaSW5mYyvZHctiQ/UOP2PfdmFEQ==",
"license": "MIT",
"funding": {
"type": "github",
@@ -1327,13 +1327,13 @@
}
},
"node_modules/@tanstack/vue-query": {
- "version": "5.100.6",
- "resolved": "https://registry.npmjs.org/@tanstack/vue-query/-/vue-query-5.100.6.tgz",
- "integrity": "sha512-aUvBla5roKKI311C4KHg+n0dHe+s9MKXwhUcro6Ajvu0L6IERoZ+/CjRqUCUgd5Y51ytuZAJl85uahgp7ViVlg==",
+ "version": "5.95.2",
+ "resolved": "https://registry.npmjs.org/@tanstack/vue-query/-/vue-query-5.95.2.tgz",
+ "integrity": "sha512-GleO0GrUPdvObtff/D3iQ5kUERQM3dM6vT5pWl4zC3ap2JO84x4SQbUa1G7czKx96lETRiHnw7ZuatSRaaZqQQ==",
"license": "MIT",
"dependencies": {
"@tanstack/match-sorter-utils": "^8.19.4",
- "@tanstack/query-core": "5.100.6",
+ "@tanstack/query-core": "5.95.2",
"@vue/devtools-api": "^6.6.3",
"vue-demi": "^0.14.10"
},
@@ -1394,53 +1394,53 @@
}
},
"node_modules/@vue/compiler-core": {
- "version": "3.5.33",
- "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.5.33.tgz",
- "integrity": "sha512-3PZLQwFw4Za3TC8t0FvTy3wI16Kt+pmwcgNZca4Pj9iWL2E72a/gZlpBtAJvEdDMdCxdG/qq0C7PN0bsJuv0Rw==",
+ "version": "3.5.31",
+ "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.5.31.tgz",
+ "integrity": "sha512-k/ueL14aNIEy5Onf0OVzR8kiqF/WThgLdFhxwa4e/KF/0qe38IwIdofoSWBTvvxQOesaz6riAFAUaYjoF9fLLQ==",
"license": "MIT",
"dependencies": {
"@babel/parser": "^7.29.2",
- "@vue/shared": "3.5.33",
+ "@vue/shared": "3.5.31",
"entities": "^7.0.1",
"estree-walker": "^2.0.2",
"source-map-js": "^1.2.1"
}
},
"node_modules/@vue/compiler-dom": {
- "version": "3.5.33",
- "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.5.33.tgz",
- "integrity": "sha512-PXq0yrfCLzzL07rbXO4awtXY1Z06LG2eu6Adg3RJFa/j3Cii217XxxLXG22N330gw7GmALCY0Z8RgXEviwgpjA==",
+ "version": "3.5.31",
+ "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.5.31.tgz",
+ "integrity": "sha512-BMY/ozS/xxjYqRFL+tKdRpATJYDTTgWSo0+AJvJNg4ig+Hgb0dOsHPXvloHQ5hmlivUqw1Yt2pPIqp4e0v1GUw==",
"license": "MIT",
"dependencies": {
- "@vue/compiler-core": "3.5.33",
- "@vue/shared": "3.5.33"
+ "@vue/compiler-core": "3.5.31",
+ "@vue/shared": "3.5.31"
}
},
"node_modules/@vue/compiler-sfc": {
- "version": "3.5.33",
- "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.5.33.tgz",
- "integrity": "sha512-UTUvRO9cY+rROrx/pvN9P5Z7FgA6QGfokUCfhQE4EnmUj3rVnK+CHI0LsEO1pg+I7//iRYMUfcNcCPe7tg0CoA==",
+ "version": "3.5.31",
+ "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.5.31.tgz",
+ "integrity": "sha512-M8wpPgR9UJ8MiRGjppvx9uWJfLV7A/T+/rL8s/y3QG3u0c2/YZgff3d6SuimKRIhcYnWg5fTfDMlz2E6seUW8Q==",
"license": "MIT",
"dependencies": {
"@babel/parser": "^7.29.2",
- "@vue/compiler-core": "3.5.33",
- "@vue/compiler-dom": "3.5.33",
- "@vue/compiler-ssr": "3.5.33",
- "@vue/shared": "3.5.33",
+ "@vue/compiler-core": "3.5.31",
+ "@vue/compiler-dom": "3.5.31",
+ "@vue/compiler-ssr": "3.5.31",
+ "@vue/shared": "3.5.31",
"estree-walker": "^2.0.2",
"magic-string": "^0.30.21",
- "postcss": "^8.5.10",
+ "postcss": "^8.5.8",
"source-map-js": "^1.2.1"
}
},
"node_modules/@vue/compiler-ssr": {
- "version": "3.5.33",
- "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.5.33.tgz",
- "integrity": "sha512-IErjYdnj1qIupG5xxiVIYiiRvDhGWV4zuh/RCrwfYpuL+HWQzeU6lCk/nF9r7olWMnjKxCAkOctT2qFWFkzb1A==",
+ "version": "3.5.31",
+ "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.5.31.tgz",
+ "integrity": "sha512-h0xIMxrt/LHOvJKMri+vdYT92BrK3HFLtDqq9Pr/lVVfE4IyKZKvWf0vJFW10Yr6nX02OR4MkJwI0c1HDa1hog==",
"license": "MIT",
"dependencies": {
- "@vue/compiler-dom": "3.5.33",
- "@vue/shared": "3.5.33"
+ "@vue/compiler-dom": "3.5.31",
+ "@vue/shared": "3.5.31"
}
},
"node_modules/@vue/devtools-api": {
@@ -1450,53 +1450,53 @@
"license": "MIT"
},
"node_modules/@vue/reactivity": {
- "version": "3.5.33",
- "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.5.33.tgz",
- "integrity": "sha512-p8UfIqyIhb0rYGlSgSBV+lPhF2iUSBcRy7enhTmPqKWadHy9kcOFYF1AejYBP9P+avnd3OBbD49DU4pLWX/94A==",
+ "version": "3.5.31",
+ "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.5.31.tgz",
+ "integrity": "sha512-DtKXxk9E/KuVvt8VxWu+6Luc9I9ETNcqR1T1oW1gf02nXaZ1kuAx58oVu7uX9XxJR0iJCro6fqBLw9oSBELo5g==",
"license": "MIT",
"dependencies": {
- "@vue/shared": "3.5.33"
+ "@vue/shared": "3.5.31"
}
},
"node_modules/@vue/runtime-core": {
- "version": "3.5.33",
- "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.5.33.tgz",
- "integrity": "sha512-UpFF45RI9//a7rvq7RdOQblb4tup7hHG9QsmIrxkFQLzQ7R8/iNQ5LE15NhLZ1/WcHMU2b47u6P33CPUelHyIQ==",
+ "version": "3.5.31",
+ "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.5.31.tgz",
+ "integrity": "sha512-AZPmIHXEAyhpkmN7aWlqjSfYynmkWlluDNPHMCZKFHH+lLtxP/30UJmoVhXmbDoP1Ng0jG0fyY2zCj1PnSSA6Q==",
"license": "MIT",
"dependencies": {
- "@vue/reactivity": "3.5.33",
- "@vue/shared": "3.5.33"
+ "@vue/reactivity": "3.5.31",
+ "@vue/shared": "3.5.31"
}
},
"node_modules/@vue/runtime-dom": {
- "version": "3.5.33",
- "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.5.33.tgz",
- "integrity": "sha512-IOxMsAOwquhfITgmOgaPYl7/j8gKUxUFoflRc+u4LxyD3+783xne8vNta1PONVCvCV9A0w7hkyEepINDqfO0tw==",
+ "version": "3.5.31",
+ "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.5.31.tgz",
+ "integrity": "sha512-xQJsNRmGPeDCJq/u813tyonNgWBFjzfVkBwDREdEWndBnGdHLHgkwNBQxLtg4zDrzKTEcnikUy1UUNecb3lJ6g==",
"license": "MIT",
"dependencies": {
- "@vue/reactivity": "3.5.33",
- "@vue/runtime-core": "3.5.33",
- "@vue/shared": "3.5.33",
+ "@vue/reactivity": "3.5.31",
+ "@vue/runtime-core": "3.5.31",
+ "@vue/shared": "3.5.31",
"csstype": "^3.2.3"
}
},
"node_modules/@vue/server-renderer": {
- "version": "3.5.33",
- "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.5.33.tgz",
- "integrity": "sha512-0xylq/8/h44lVG0pZFknv1XIdEgymq2E9n59uTWJBG+dIgiT0TMCSsxrN7nO16Z0MU0MPjFcguBbZV8Itk52Hw==",
+ "version": "3.5.31",
+ "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.5.31.tgz",
+ "integrity": "sha512-GJuwRvMcdZX/CriUnyIIOGkx3rMV3H6sOu0JhdKbduaeCji6zb60iOGMY7tFoN24NfsUYoFBhshZtGxGpxO4iA==",
"license": "MIT",
"dependencies": {
- "@vue/compiler-ssr": "3.5.33",
- "@vue/shared": "3.5.33"
+ "@vue/compiler-ssr": "3.5.31",
+ "@vue/shared": "3.5.31"
},
"peerDependencies": {
- "vue": "3.5.33"
+ "vue": "3.5.31"
}
},
"node_modules/@vue/shared": {
- "version": "3.5.33",
- "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.5.33.tgz",
- "integrity": "sha512-5vR2QIlmaLG77Ygd4pMP6+SGQ5yox9VhtnbDWTy9DzMzdmeLxZ1QqxrywEZ9sa1AVubfIJyaCG3ytyWU81ufcQ==",
+ "version": "3.5.31",
+ "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.5.31.tgz",
+ "integrity": "sha512-nBxuiuS9Lj5bPkPbWogPUnjxxWpkRniX7e5UBQDWl6Fsf4roq9wwV+cR7ezQ4zXswNvPIlsdj1slcLB7XCsRAw==",
"license": "MIT"
},
"node_modules/@vueuse/core": {
@@ -1608,9 +1608,9 @@
}
},
"node_modules/brace-expansion": {
- "version": "2.1.0",
- "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz",
- "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==",
+ "version": "2.0.3",
+ "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz",
+ "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -1715,9 +1715,9 @@
}
},
"node_modules/element-plus": {
- "version": "2.13.7",
- "resolved": "https://registry.npmjs.org/element-plus/-/element-plus-2.13.7.tgz",
- "integrity": "sha512-XdHATFZOyzVFL1DaHQ90IOJQSg9UnSAV+bhDW+YB5UoZ0Hxs50mwqjqfwXkuwpSag+VXXizVcErBR6Movo5daw==",
+ "version": "2.13.6",
+ "resolved": "https://registry.npmjs.org/element-plus/-/element-plus-2.13.6.tgz",
+ "integrity": "sha512-XHgwXr8Fjz6i+6BaqFhAbae/dJbG7bBAAlHrY3pWL7dpj+JcqcOyKYt4Oy5KP86FQwS1k4uIZDjCx2FyUR5lDg==",
"license": "MIT",
"dependencies": {
"@ctrl/tinycolor": "^4.2.0",
@@ -1752,16 +1752,6 @@
"url": "https://github.com/fb55/entities?sponsor=1"
}
},
- "node_modules/es-errors": {
- "version": "1.3.0",
- "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
- "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
- "dev": true,
- "license": "MIT",
- "engines": {
- "node": ">= 0.4"
- }
- },
"node_modules/esbuild": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz",
@@ -1934,9 +1924,9 @@
"optional": true
},
"node_modules/hasown": {
- "version": "2.0.3",
- "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz",
- "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==",
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
+ "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -2052,9 +2042,9 @@
}
},
"node_modules/jsonfile": {
- "version": "6.2.1",
- "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz",
- "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==",
+ "version": "6.2.0",
+ "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz",
+ "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==",
"license": "MIT",
"dependencies": {
"universalify": "^2.0.0"
@@ -2333,9 +2323,9 @@
}
},
"node_modules/postcss": {
- "version": "8.5.12",
- "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.12.tgz",
- "integrity": "sha512-W62t/Se6rA0Az3DfCL0AqJwXuKwBeYg6nOaIgzP+xZ7N5BFCI7DYi1qs6ygUYT6rvfi6t9k65UMLJC+PHZpDAA==",
+ "version": "8.5.8",
+ "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz",
+ "integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==",
"funding": [
{
"type": "opencollective",
@@ -2361,9 +2351,9 @@
}
},
"node_modules/prettier": {
- "version": "3.8.3",
- "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.3.tgz",
- "integrity": "sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==",
+ "version": "3.8.1",
+ "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.1.tgz",
+ "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==",
"dev": true,
"license": "MIT",
"bin": {
@@ -2455,13 +2445,12 @@
"license": "MIT"
},
"node_modules/resolve": {
- "version": "1.22.12",
- "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz",
- "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==",
+ "version": "1.22.11",
+ "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz",
+ "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==",
"dev": true,
"license": "MIT",
"dependencies": {
- "es-errors": "^1.3.0",
"is-core-module": "^2.16.1",
"path-parse": "^1.0.7",
"supports-preserve-symlinks-flag": "^1.0.0"
@@ -2488,9 +2477,9 @@
}
},
"node_modules/rollup": {
- "version": "4.60.2",
- "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.2.tgz",
- "integrity": "sha512-J9qZyW++QK/09NyN/zeO0dG/1GdGfyp9lV8ajHnRVLfo/uFsbji5mHnDgn/qYdUHyCkM2N+8VyspgZclfAh0eQ==",
+ "version": "4.60.0",
+ "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.0.tgz",
+ "integrity": "sha512-yqjxruMGBQJ2gG4HtjZtAfXArHomazDHoFwFFmZZl0r7Pdo7qCIXKqKHZc8yeoMgzJJ+pO6pEEHa+V7uzWlrAQ==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -2504,31 +2493,31 @@
"npm": ">=8.0.0"
},
"optionalDependencies": {
- "@rollup/rollup-android-arm-eabi": "4.60.2",
- "@rollup/rollup-android-arm64": "4.60.2",
- "@rollup/rollup-darwin-arm64": "4.60.2",
- "@rollup/rollup-darwin-x64": "4.60.2",
- "@rollup/rollup-freebsd-arm64": "4.60.2",
- "@rollup/rollup-freebsd-x64": "4.60.2",
- "@rollup/rollup-linux-arm-gnueabihf": "4.60.2",
- "@rollup/rollup-linux-arm-musleabihf": "4.60.2",
- "@rollup/rollup-linux-arm64-gnu": "4.60.2",
- "@rollup/rollup-linux-arm64-musl": "4.60.2",
- "@rollup/rollup-linux-loong64-gnu": "4.60.2",
- "@rollup/rollup-linux-loong64-musl": "4.60.2",
- "@rollup/rollup-linux-ppc64-gnu": "4.60.2",
- "@rollup/rollup-linux-ppc64-musl": "4.60.2",
- "@rollup/rollup-linux-riscv64-gnu": "4.60.2",
- "@rollup/rollup-linux-riscv64-musl": "4.60.2",
- "@rollup/rollup-linux-s390x-gnu": "4.60.2",
- "@rollup/rollup-linux-x64-gnu": "4.60.2",
- "@rollup/rollup-linux-x64-musl": "4.60.2",
- "@rollup/rollup-openbsd-x64": "4.60.2",
- "@rollup/rollup-openharmony-arm64": "4.60.2",
- "@rollup/rollup-win32-arm64-msvc": "4.60.2",
- "@rollup/rollup-win32-ia32-msvc": "4.60.2",
- "@rollup/rollup-win32-x64-gnu": "4.60.2",
- "@rollup/rollup-win32-x64-msvc": "4.60.2",
+ "@rollup/rollup-android-arm-eabi": "4.60.0",
+ "@rollup/rollup-android-arm64": "4.60.0",
+ "@rollup/rollup-darwin-arm64": "4.60.0",
+ "@rollup/rollup-darwin-x64": "4.60.0",
+ "@rollup/rollup-freebsd-arm64": "4.60.0",
+ "@rollup/rollup-freebsd-x64": "4.60.0",
+ "@rollup/rollup-linux-arm-gnueabihf": "4.60.0",
+ "@rollup/rollup-linux-arm-musleabihf": "4.60.0",
+ "@rollup/rollup-linux-arm64-gnu": "4.60.0",
+ "@rollup/rollup-linux-arm64-musl": "4.60.0",
+ "@rollup/rollup-linux-loong64-gnu": "4.60.0",
+ "@rollup/rollup-linux-loong64-musl": "4.60.0",
+ "@rollup/rollup-linux-ppc64-gnu": "4.60.0",
+ "@rollup/rollup-linux-ppc64-musl": "4.60.0",
+ "@rollup/rollup-linux-riscv64-gnu": "4.60.0",
+ "@rollup/rollup-linux-riscv64-musl": "4.60.0",
+ "@rollup/rollup-linux-s390x-gnu": "4.60.0",
+ "@rollup/rollup-linux-x64-gnu": "4.60.0",
+ "@rollup/rollup-linux-x64-musl": "4.60.0",
+ "@rollup/rollup-openbsd-x64": "4.60.0",
+ "@rollup/rollup-openharmony-arm64": "4.60.0",
+ "@rollup/rollup-win32-arm64-msvc": "4.60.0",
+ "@rollup/rollup-win32-ia32-msvc": "4.60.0",
+ "@rollup/rollup-win32-x64-gnu": "4.60.0",
+ "@rollup/rollup-win32-x64-msvc": "4.60.0",
"fsevents": "~2.3.2"
}
},
@@ -2563,9 +2552,9 @@
"license": "MIT"
},
"node_modules/sass": {
- "version": "1.99.0",
- "resolved": "https://registry.npmjs.org/sass/-/sass-1.99.0.tgz",
- "integrity": "sha512-kgW13M54DUB7IsIRM5LvJkNlpH+WhMpooUcaWGFARkF1Tc82v9mIWkCbCYf+MBvpIUBSeSOTilpZjEPr2VYE6Q==",
+ "version": "1.98.0",
+ "resolved": "https://registry.npmjs.org/sass/-/sass-1.98.0.tgz",
+ "integrity": "sha512-+4N/u9dZ4PrgzGgPlKnaaRQx64RO0JBKs9sDhQ2pLgN6JQZ25uPQZKQYaBJU48Kd5BxgXoJ4e09Dq7nMcOUW3A==",
"license": "MIT",
"dependencies": {
"chokidar": "^4.0.0",
@@ -2652,14 +2641,14 @@
}
},
"node_modules/tinyglobby": {
- "version": "0.2.16",
- "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz",
- "integrity": "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==",
+ "version": "0.2.15",
+ "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz",
+ "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"fdir": "^6.5.0",
- "picomatch": "^4.0.4"
+ "picomatch": "^4.0.3"
},
"engines": {
"node": ">=12.0.0"
@@ -2753,14 +2742,14 @@
}
},
"node_modules/unimport/node_modules/local-pkg/node_modules/pkg-types": {
- "version": "2.3.1",
- "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-2.3.1.tgz",
- "integrity": "sha512-y+ichcgc2LrADuhLNAx8DFjVfgz91pRxfZdI3UDhxHvcVEZsenLO+7XaU5vOp0u/7V/wZ+plyuQxtrDlZJ+yeg==",
+ "version": "2.3.0",
+ "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-2.3.0.tgz",
+ "integrity": "sha512-SIqCzDRg0s9npO5XQ3tNZioRY1uK06lA41ynBC1YmFTmnY6FjUjVt6s4LoADmwoig1qqD0oK8h1p/8mlMx8Oig==",
"dev": true,
"license": "MIT",
"dependencies": {
- "confbox": "^0.2.4",
- "exsolve": "^1.0.8",
+ "confbox": "^0.2.2",
+ "exsolve": "^1.0.7",
"pathe": "^2.0.3"
}
},
@@ -2931,9 +2920,9 @@
"license": "MIT"
},
"node_modules/vite": {
- "version": "6.4.2",
- "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.2.tgz",
- "integrity": "sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ==",
+ "version": "6.4.1",
+ "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz",
+ "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -3006,16 +2995,16 @@
}
},
"node_modules/vue": {
- "version": "3.5.33",
- "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.33.tgz",
- "integrity": "sha512-1AgChhx5w3ALgT4oK3acm2Es/7jyZhWSVUfs3rOBlGQC0rjEDkS7G4lWlJJGGNQD+BV3reCwbQrOe1mPNwKHBQ==",
+ "version": "3.5.31",
+ "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.31.tgz",
+ "integrity": "sha512-iV/sU9SzOlmA/0tygSmjkEN6Jbs3nPoIPFhCMLD2STrjgOU8DX7ZtzMhg4ahVwf5Rp9KoFzcXeB1ZrVbLBp5/Q==",
"license": "MIT",
"dependencies": {
- "@vue/compiler-dom": "3.5.33",
- "@vue/compiler-sfc": "3.5.33",
- "@vue/runtime-dom": "3.5.33",
- "@vue/server-renderer": "3.5.33",
- "@vue/shared": "3.5.33"
+ "@vue/compiler-dom": "3.5.31",
+ "@vue/compiler-sfc": "3.5.31",
+ "@vue/runtime-dom": "3.5.31",
+ "@vue/server-renderer": "3.5.31",
+ "@vue/shared": "3.5.31"
},
"peerDependencies": {
"typescript": "*"
@@ -3027,9 +3016,9 @@
}
},
"node_modules/vue-component-type-helpers": {
- "version": "3.2.7",
- "resolved": "https://registry.npmjs.org/vue-component-type-helpers/-/vue-component-type-helpers-3.2.7.tgz",
- "integrity": "sha512-+gPp5YGmhfsj1IN+xUo7y0fb4clfnOiiUA39y07yW1VzCRjzVgwLbtmdWlghh7mXrPsEaYc7rrIir/HT6C8vYQ==",
+ "version": "3.2.6",
+ "resolved": "https://registry.npmjs.org/vue-component-type-helpers/-/vue-component-type-helpers-3.2.6.tgz",
+ "integrity": "sha512-O02tnvIfOQVmnvoWwuSydwRoHjZVt8UEBR+2p4rT35p8GAy5VTlWP8o5qXfJR/GWCN0nVZoYWsVUvx2jwgdBmQ==",
"license": "MIT"
},
"node_modules/vue-demi": {
diff --git a/ui/src/components/CodeMirror/bydbql-mode.js b/ui/src/components/CodeMirror/bydbql-mode.js
index eb125d3d7..8aac7cb9b 100644
--- a/ui/src/components/CodeMirror/bydbql-mode.js
+++ b/ui/src/components/CodeMirror/bydbql-mode.js
@@ -103,6 +103,15 @@ CodeMirror.defineMode('bydbql', function (config) {
},
token: function (stream, state) {
+ const prefixedKeywordMatch = stream.match(/^--([A-Za-z_]\w*)/, false);
+ if (prefixedKeywordMatch) {
+ const upperWord = prefixedKeywordMatch[1].toUpperCase();
+ if (entityTypes[upperWord] || bydbqlKeywords[upperWord]) {
+ stream.match(/^--[A-Za-z_]\w*/);
+ return entityTypes[upperWord] ? 'entity-type' : 'keyword';
+ }
+ }
+
const style = sqlMode.token(stream, state.sqlState);
if (style === 'comment' || style === 'string') {
return style;
diff --git a/ui/src/components/Property/PropertyRead.vue b/ui/src/components/Property/PropertyRead.vue
index ffdf34ee7..275026921 100644
--- a/ui/src/components/Property/PropertyRead.vue
+++ b/ui/src/components/Property/PropertyRead.vue
@@ -40,13 +40,30 @@
tableData: [],
name: route.params.name,
});
- const yamlCode = ref(`name: ${data.name}
-limit: 10`);
+ const yamlCode = ref(buildSearchYaml(data.name, 10));
const showTracesDialog = ref(false);
const traceData = ref(null);
- const getProperties = async (params) => {
+
+ function buildSearchYaml(name, limit) {
+ return `name: ${name}
+limit: ${limit}`;
+ }
+
+ function normalizeSearchParams(params = {}) {
+ return {
+ ...params,
+ name: params.name ?? data.name,
+ limit: params.limit ?? 10,
+ };
+ }
+
+ const getProperties = async (params, resetYaml = false) => {
+ const normalizedParams = normalizeSearchParams(params);
+ if (resetYaml) {
+ yamlCode.value = buildSearchYaml(normalizedParams.name, normalizedParams.limit);
+ }
$loadingCreate();
- const res = await fetchProperties({ groups: [data.group], name: data.name, limit: 10, ...params });
+ const res = await fetchProperties({ groups: [data.group], ...normalizedParams });
$loadingClose();
if (res.error) {
ElMessage({
@@ -82,8 +99,12 @@ limit: 10`);
});
}
+ function refreshProperties() {
+ getProperties(undefined, true);
+ }
+
onMounted(() => {
- getProperties();
+ getProperties(undefined, true);
});
watch(
() => route.params,
@@ -91,8 +112,7 @@ limit: 10`);
const { group, name } = route.params;
data.name = name;
data.group = group;
- yamlCode.value = `name: ${data.name}
-limit: 10`;
+ yamlCode.value = buildSearchYaml(data.name, 10);
getProperties();
},
);
@@ -105,7 +125,7 @@ limit: 10`;
</template>
<div class="button-group-operator">
<el-button size="small" :icon="Search" @click="searchProperties" plain />
- <el-button size="small" :icon="RefreshRight" @click="getProperties" plain />
+ <el-button size="small" :icon="RefreshRight" @click="refreshProperties" plain />
</div>
<CodeMirror ref="yamlRef" v-model="yamlCode" mode="yaml" style="height: 200px" :lint="true" />
<div style="margin-top: 20px; margin-bottom: 10px; display: flex; justify-content: flex-end">
diff --git a/ui/src/components/Read/index.vue b/ui/src/components/Read/index.vue
index 753c67d82..51a2a9dfb 100644
--- a/ui/src/components/Read/index.vue
+++ b/ui/src/components/Read/index.vue
@@ -66,6 +66,7 @@
],
},
});
+ const filterConfigBaseKeys = ['groups', 'name', 'offset', 'limit', 'stages', 'projection'];
const data = reactive({
fields: [],
tableFields: [],
@@ -282,6 +283,11 @@ orderBy:
}
function handleCodeData() {
const json = yamlToJson(data.code).data;
+ Object.keys(filterConfig).forEach((key) => {
+ if (!filterConfigBaseKeys.includes(key)) {
+ delete filterConfig[key];
+ }
+ });
filterConfig.offset = json.offset !== undefined ? json.offset : 0;
filterConfig.limit = json.limit !== undefined ? json.limit : 10;
delete filterConfig.timeRange;
@@ -327,6 +333,12 @@ orderBy:
});
});
}
+
+ function refreshTableData() {
+ initCode();
+ handleCodeData();
+ }
+
function changeDatePicker() {
const json = yamlToJson(data.code);
if (!json.data.hasOwnProperty('timeRange')) {
@@ -404,7 +416,7 @@ orderBy:
</el-col>
<el-col :span="8">
<div class="flex align-item-center justify-end" style="height: 30px">
- <el-button :icon="RefreshRight" @click="getTableData" plain />
+ <el-button :icon="RefreshRight" @click="refreshTableData" plain />
</div>
</el-col>
</el-row>
(skywalking-banyandb) 05/14: fix(measure,stream,trace): wait until all mem parts flushed in file_snapshot tests (#1098)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit ce600a7029dd1c18f52e6eac430f54a69615b81f
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Tue Apr 28 15:42:20 2026 +0800
fix(measure,stream,trace): wait until all mem parts flushed in file_snapshot tests (#1098)
---
CHANGES.md | 6 ++++
banyand/measure/query_test.go | 64 ++++++++++++++++++------------------
banyand/stream/block_scanner_test.go | 42 ++++++++++++++---------
banyand/stream/query_by_idx_test.go | 22 ++++---------
banyand/trace/query_test.go | 42 ++++++++++++++---------
5 files changed, 96 insertions(+), 80 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 1a0884ddd..eac054fab 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -22,6 +22,12 @@ Release Notes.
- Add validation for MATCH and IN conditions in inverted index query builder, and handle nil OR branch when all entities are specific.
- Fix wrong backup path of schema property.
- Fix lifecycle migration failure when the target stage has `close: true`.
+- Fix stale sync request blocking watch session channel, causing repeated "channel full, skipping session" errors when a watch stream is in backoff.
+- Fix nil pointer panic in disk monitor when group schema is not yet initialized during early startup, and ensure monitor loop survives recovered panics.
+- Fix `FileSystemError` not satisfying `errors.Is(err, io/fs.ErrNotExist)`, which prevented the segment controller from cleaning up half-born segment directories and left groups in a permanent zombie state after a crash or partial sync.
+- Fix lifecycle migration panic when a stream shard's snapshot has no element index (`idx/`) directory.
+- Avoid FODC lifecycle inspection failing on busy data nodes by raising the per-broadcast `CollectDataInfo` / `CollectLiaisonInfo` deadline from 5s to 30s and parallelizing per-group inspection in the cluster-internal `InspectAll`.
+- Fix flaky `file_snapshot` subtest in measure/stream/trace by waiting until every introduced mem part has been flushed to disk, instead of only checking the latest snapshot creator.
### Chores
diff --git a/banyand/measure/query_test.go b/banyand/measure/query_test.go
index 66ab60779..c72a19719 100644
--- a/banyand/measure/query_test.go
+++ b/banyand/measure/query_test.go
@@ -1335,23 +1335,13 @@ func TestQueryResult(t *testing.T) {
tst.mustAddDataPoints(dps)
time.Sleep(100 * time.Millisecond)
}
- // wait until the introducer is done
+ // wait until every introduced mem part has been flushed to disk;
+ // inspecting only snapshot.creator races with concurrent introductions
+ // (a flush of part N can land while part N+1 is still a mem part).
if len(tt.dpsList) > 0 {
- for {
- snp := tst.currentSnapshot()
- if snp == nil {
- time.Sleep(100 * time.Millisecond)
- continue
- }
- if snp.creator == snapshotCreatorMemPart {
- snp.decRef()
- time.Sleep(100 * time.Millisecond)
- continue
- }
- snp.decRef()
- tst.Close()
- break
- }
+ require.Eventually(t, allPartsFlushed(tst), 30*time.Second, 100*time.Millisecond,
+ "mem parts not flushed to disk in time")
+ tst.Close()
}
// reopen the table
@@ -1368,6 +1358,26 @@ func TestQueryResult(t *testing.T) {
}
}
+// allPartsFlushed returns a predicate that is true once tst's current
+// snapshot exists and every part is file-backed (no mp != nil).
+// It is used by file_snapshot tests to wait until every introduced mem
+// part has been flushed before closing/reopening the table.
+func allPartsFlushed(tst *tsTable) func() bool {
+ return func() bool {
+ snp := tst.currentSnapshot()
+ if snp == nil {
+ return false
+ }
+ defer snp.decRef()
+ for _, pw := range snp.parts {
+ if pw.mp != nil {
+ return false
+ }
+ }
+ return true
+ }
+}
+
func TestQueryResult_QuotaExceeded(t *testing.T) {
tests := []struct {
wantErr error
@@ -1438,23 +1448,13 @@ func TestQueryResult_QuotaExceeded(t *testing.T) {
tst.mustAddDataPoints(dps)
time.Sleep(100 * time.Millisecond)
}
- // wait until the introducer is done
+ // wait until every introduced mem part has been flushed to disk;
+ // inspecting only snapshot.creator races with concurrent introductions
+ // (a flush of part N can land while part N+1 is still a mem part).
if len(tt.dpsList) > 0 {
- for {
- snp := tst.currentSnapshot()
- if snp == nil {
- time.Sleep(100 * time.Millisecond)
- continue
- }
- if snp.creator == snapshotCreatorMemPart {
- snp.decRef()
- time.Sleep(100 * time.Millisecond)
- continue
- }
- snp.decRef()
- tst.Close()
- break
- }
+ require.Eventually(t, allPartsFlushed(tst), 30*time.Second, 100*time.Millisecond,
+ "mem parts not flushed to disk in time")
+ tst.Close()
}
// reopen the table
diff --git a/banyand/stream/block_scanner_test.go b/banyand/stream/block_scanner_test.go
index b7248b7c6..ac592b9d7 100644
--- a/banyand/stream/block_scanner_test.go
+++ b/banyand/stream/block_scanner_test.go
@@ -87,23 +87,13 @@ func TestBlockScanner_QuotaExceeded(t *testing.T) {
tst.mustAddElements(es)
time.Sleep(100 * time.Millisecond)
}
- // wait until the introducer is done
+ // wait until every introduced mem part has been flushed to disk;
+ // inspecting only snapshot.creator races with concurrent introductions
+ // (a flush of part N can land while part N+1 is still a mem part).
if len(tt.esList) > 0 {
- for {
- snp := tst.currentSnapshot()
- if snp == nil {
- time.Sleep(100 * time.Millisecond)
- continue
- }
- if snp.creator == snapshotCreatorMemPart {
- snp.decRef()
- time.Sleep(100 * time.Millisecond)
- continue
- }
- snp.decRef()
- tst.Close()
- break
- }
+ require.Eventually(t, allPartsFlushed(tst), 30*time.Second, 100*time.Millisecond,
+ "mem parts not flushed to disk in time")
+ tst.Close()
}
// reopen the table
@@ -204,3 +194,23 @@ func TestBlockScanner_QuotaExceeded(t *testing.T) {
})
}
}
+
+// allPartsFlushed returns a predicate that is true once tst's current
+// snapshot exists and every part is file-backed (no mp != nil).
+// It is used by file_snapshot tests to wait until every introduced mem
+// part has been flushed before closing/reopening the table.
+func allPartsFlushed(tst *tsTable) func() bool {
+ return func() bool {
+ snp := tst.currentSnapshot()
+ if snp == nil {
+ return false
+ }
+ defer snp.decRef()
+ for _, pw := range snp.parts {
+ if pw.mp != nil {
+ return false
+ }
+ }
+ return true
+ }
+}
diff --git a/banyand/stream/query_by_idx_test.go b/banyand/stream/query_by_idx_test.go
index 526546593..4a695bd8e 100644
--- a/banyand/stream/query_by_idx_test.go
+++ b/banyand/stream/query_by_idx_test.go
@@ -85,23 +85,13 @@ func TestQueryResult_QuotaExceeded(t *testing.T) {
tst.mustAddElements(es)
time.Sleep(100 * time.Millisecond)
}
- // wait until the introducer is done
+ // wait until every introduced mem part has been flushed to disk;
+ // inspecting only snapshot.creator races with concurrent introductions
+ // (a flush of part N can land while part N+1 is still a mem part).
if len(tt.esList) > 0 {
- for {
- snp := tst.currentSnapshot()
- if snp == nil {
- time.Sleep(100 * time.Millisecond)
- continue
- }
- if snp.creator == snapshotCreatorMemPart {
- snp.decRef()
- time.Sleep(100 * time.Millisecond)
- continue
- }
- snp.decRef()
- tst.Close()
- break
- }
+ require.Eventually(t, allPartsFlushed(tst), 30*time.Second, 100*time.Millisecond,
+ "mem parts not flushed to disk in time")
+ tst.Close()
}
// reopen the table
diff --git a/banyand/trace/query_test.go b/banyand/trace/query_test.go
index f318d508d..54abde693 100644
--- a/banyand/trace/query_test.go
+++ b/banyand/trace/query_test.go
@@ -202,23 +202,13 @@ func TestQueryResult(t *testing.T) {
tst.mustAddTraces(traces, nil)
time.Sleep(100 * time.Millisecond)
}
- // wait until the introducer is done
+ // wait until every introduced mem part has been flushed to disk;
+ // inspecting only snapshot.creator races with concurrent introductions
+ // (a flush of part N can land while part N+1 is still a mem part).
if len(tt.tracesList) > 0 {
- for {
- snp := tst.currentSnapshot()
- if snp == nil {
- time.Sleep(100 * time.Millisecond)
- continue
- }
- if snp.creator == snapshotCreatorMemPart {
- snp.decRef()
- time.Sleep(100 * time.Millisecond)
- continue
- }
- snp.decRef()
- tst.Close()
- break
- }
+ require.Eventually(t, allPartsFlushed(tst), 30*time.Second, 100*time.Millisecond,
+ "mem parts not flushed to disk in time")
+ tst.Close()
}
// reopen the table
@@ -457,3 +447,23 @@ func TestQueryResultMultipleBatches(t *testing.T) {
})
}
}
+
+// allPartsFlushed returns a predicate that is true once tst's current
+// snapshot exists and every part is file-backed (no mp != nil).
+// It is used by file_snapshot tests to wait until every introduced mem
+// part has been flushed before closing/reopening the table.
+func allPartsFlushed(tst *tsTable) func() bool {
+ return func() bool {
+ snp := tst.currentSnapshot()
+ if snp == nil {
+ return false
+ }
+ defer snp.decRef()
+ for _, pw := range snp.parts {
+ if pw.mp != nil {
+ return false
+ }
+ }
+ return true
+ }
+}
(skywalking-banyandb) 14/14: Support multiple types for the same tag in a single trace or sidx part (#1066)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit c9961ab39ab8bb9680117772ba05131506b63809
Author: Huang Youliang <bu...@gmail.com>
AuthorDate: Tue Apr 21 10:06:35 2026 +0800
Support multiple types for the same tag in a single trace or sidx part (#1066)
* Support multiple types for the same tag in a single part
Co-authored-by: Gao Hongtao <ha...@gmail.com>
---
api/validate/validate.go | 26 ++++++
banyand/internal/sidx/block.go | 2 +
banyand/internal/sidx/interfaces.go | 38 +++++----
banyand/internal/sidx/merge.go | 84 +++++++++++++++++--
banyand/internal/sidx/merge_test.go | 121 ++++++++++++++++++++++++++
banyand/internal/sidx/query_result.go | 26 ++++--
banyand/internal/sidx/scan_query.go | 17 ++--
banyand/internal/sidx/tag.go | 79 +++++++++++++++--
banyand/trace/block.go | 31 +++++--
banyand/trace/merger.go | 84 +++++++++++++++++--
banyand/trace/merger_bench_test.go | 8 +-
banyand/trace/merger_test.go | 154 +++++++++++++++++++++++++++++++++-
banyand/trace/query.go | 15 ++--
banyand/trace/svc_standalone.go | 1 +
banyand/trace/tag.go | 53 ++++++++++++
banyand/trace/tag_test.go | 19 +++++
banyand/trace/trace_suite_test.go | 2 +-
17 files changed, 687 insertions(+), 73 deletions(-)
diff --git a/api/validate/validate.go b/api/validate/validate.go
index 61e7b5d75..946c99061 100644
--- a/api/validate/validate.go
+++ b/api/validate/validate.go
@@ -20,11 +20,22 @@ package validate
import (
"errors"
+ "fmt"
+ "strings"
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
)
+const reservedTagSeparator = "#"
+
+func validateTagName(name string) error {
+ if strings.Contains(name, reservedTagSeparator) {
+ return fmt.Errorf("tag name %q must not contain reserved character %q", name, reservedTagSeparator)
+ }
+ return nil
+}
+
// Group validates the provided Group object.
func Group(group *commonv1.Group) error {
if group.Metadata == nil {
@@ -188,16 +199,28 @@ func Trace(trace *databasev1.Trace) error {
if trace.TraceIdTagName == "" {
return errors.New("trace_id_tag_name is empty")
}
+ if err := validateTagName(trace.TraceIdTagName); err != nil {
+ return err
+ }
if trace.SpanIdTagName == "" {
return errors.New("span_id_tag_name is empty")
}
+ if err := validateTagName(trace.SpanIdTagName); err != nil {
+ return err
+ }
if trace.TimestampTagName == "" {
return errors.New("timestamp_tag_name is empty")
}
+ if err := validateTagName(trace.TimestampTagName); err != nil {
+ return err
+ }
for i := range trace.Tags {
if trace.Tags[i].Name == "" {
return errors.New("trace tag name is empty")
}
+ if err := validateTagName(trace.Tags[i].Name); err != nil {
+ return err
+ }
if trace.Tags[i].Type == databasev1.TagType_TAG_TYPE_UNSPECIFIED {
return errors.New("trace tag type is unspecified")
}
@@ -236,6 +259,9 @@ func tagFamily(tagFamilies []*databasev1.TagFamilySpec) error {
if tagFamilies[i].Tags[j].Name == "" {
return errors.New("tag name is empty")
}
+ if err := validateTagName(tagFamilies[i].Tags[j].Name); err != nil {
+ return err
+ }
if tagFamilies[i].Tags[j].Type == databasev1.TagType_TAG_TYPE_UNSPECIFIED {
return errors.New("tag type is unspecified")
}
diff --git a/banyand/internal/sidx/block.go b/banyand/internal/sidx/block.go
index c240dbbfc..3522b474b 100644
--- a/banyand/internal/sidx/block.go
+++ b/banyand/internal/sidx/block.go
@@ -482,6 +482,8 @@ func fastTagAppend(bi, b *blockPointer, offset int) error {
if _, exists := b.tags[t.name]; !exists {
return fmt.Errorf("unexpected tag name for tag %q", t.name)
}
+ }
+ for _, t := range bi.tags {
assertIdxAndOffset(t.name, len(b.tags[t.name].values), b.idx, offset)
bi.tags[t.name].values = append(bi.tags[t.name].values, b.tags[t.name].values[b.idx:offset]...)
}
diff --git a/banyand/internal/sidx/interfaces.go b/banyand/internal/sidx/interfaces.go
index 058c8626f..215328507 100644
--- a/banyand/internal/sidx/interfaces.go
+++ b/banyand/internal/sidx/interfaces.go
@@ -105,16 +105,17 @@ type WriteRequest struct {
// QueryRequest specifies parameters for a query operation, following StreamQueryOptions pattern.
type QueryRequest struct {
- Filter index.Filter
- TagFilter model.TagFilterMatcher
- Order *index.OrderBy
- MinKey *int64
- MaxKey *int64
- MinTimestamp *int64
- MaxTimestamp *int64
- SeriesIDs []common.SeriesID
- TagProjection []model.TagProjection
- MaxBatchSize int
+ Filter index.Filter
+ TagFilter model.TagFilterMatcher
+ Order *index.OrderBy
+ MinKey *int64
+ MaxKey *int64
+ SchemaTagTypes map[string]pbv1.ValueType
+ MinTimestamp *int64
+ MaxTimestamp *int64
+ SeriesIDs []common.SeriesID
+ TagProjection []model.TagProjection
+ MaxBatchSize int
}
// ScanProgressFunc is a callback for reporting scan progress.
@@ -127,14 +128,15 @@ type ScanProgressFunc func(currentPart, totalParts int, rowsFound int)
//
//nolint:govet // struct layout optimized for readability; 64 bytes is acceptable
type ScanQueryRequest struct {
- TagFilter model.TagFilterMatcher
- OnProgress ScanProgressFunc
- MinKey *int64
- MaxKey *int64
- MinTimestamp *int64
- MaxTimestamp *int64
- TagProjection []model.TagProjection
- MaxBatchSize int
+ TagFilter model.TagFilterMatcher
+ SchemaTagTypes map[string]pbv1.ValueType
+ OnProgress ScanProgressFunc
+ MinKey *int64
+ MaxKey *int64
+ MinTimestamp *int64
+ MaxTimestamp *int64
+ TagProjection []model.TagProjection
+ MaxBatchSize int
}
// QueryResponse contains a batch of query results and execution metadata.
diff --git a/banyand/internal/sidx/merge.go b/banyand/internal/sidx/merge.go
index ac901a2b5..83c18f3a4 100644
--- a/banyand/internal/sidx/merge.go
+++ b/banyand/internal/sidx/merge.go
@@ -19,9 +19,11 @@ package sidx
import (
"fmt"
+ "io"
"github.com/apache/skywalking-banyandb/pkg/encoding"
"github.com/apache/skywalking-banyandb/pkg/fs"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
var (
@@ -80,6 +82,7 @@ func (s *sidx) mergeParts(fileSystem fs.FileSystem, closeCh <-chan struct{}, par
return nil, errNoPartToMerge
}
dstPath := partPath(root, partID)
+ conflictTags := collectConflictTags(parts)
var totalSize int64
pii := make([]*partMergeIter, 0, len(parts))
for i := range parts {
@@ -94,7 +97,7 @@ func (s *sidx) mergeParts(fileSystem fs.FileSystem, closeCh <-chan struct{}, par
bw := generateBlockWriter()
bw.mustInitForFilePart(fileSystem, dstPath, shouldCache)
- pm, err := mergeBlocks(closeCh, bw, br)
+ pm, err := mergeBlocks(closeCh, bw, br, conflictTags)
releaseBlockWriter(bw)
releaseBlockReader(br)
for i := range pii {
@@ -133,7 +136,7 @@ func (s *sidx) mergeParts(fileSystem fs.FileSystem, closeCh <-chan struct{}, par
return newPartWrapper(nil, p), nil
}
-func mergeBlocks(closeCh <-chan struct{}, bw *blockWriter, br *blockReader) (*partMetadata, error) {
+func mergeBlocks(closeCh <-chan struct{}, bw *blockWriter, br *blockReader, conflictTags map[string]struct{}) (*partMetadata, error) {
pendingBlockIsEmpty := true
pendingBlock := generateBlockPointer()
defer releaseBlockPointer(pendingBlock)
@@ -151,6 +154,10 @@ func mergeBlocks(closeCh <-chan struct{}, bw *blockWriter, br *blockReader) (*pa
decoder = nil
}
}
+ loadAndRename := func() {
+ br.loadBlockData(getDecoder())
+ renameConflictTags(&br.block.block, conflictTags)
+ }
for br.nextBlockMetadata() {
select {
case <-closeCh:
@@ -160,7 +167,7 @@ func mergeBlocks(closeCh <-chan struct{}, bw *blockWriter, br *blockReader) (*pa
b := br.block
if pendingBlockIsEmpty {
- br.loadBlockData(getDecoder())
+ loadAndRename()
pendingBlock.copyFrom(b)
pendingBlockIsEmpty = false
continue
@@ -171,7 +178,7 @@ func mergeBlocks(closeCh <-chan struct{}, bw *blockWriter, br *blockReader) (*pa
pendingBlock.block.uncompressedSizeBytes() >= maxUncompressedBlockSize {
bw.mustWriteBlock(pendingBlock.bm.seriesID, &pendingBlock.block)
releaseDecoder()
- br.loadBlockData(getDecoder())
+ loadAndRename()
pendingBlock.copyFrom(b)
continue
}
@@ -182,7 +189,7 @@ func mergeBlocks(closeCh <-chan struct{}, bw *blockWriter, br *blockReader) (*pa
}
tmpBlock.reset()
tmpBlock.bm.seriesID = b.bm.seriesID
- br.loadBlockData(getDecoder())
+ loadAndRename()
mergeTwoBlocks(tmpBlock, pendingBlock, b)
if len(tmpBlock.userKeys) <= maxBlockLength && tmpBlock.block.uncompressedSizeBytes() <= maxUncompressedBlockSize {
if len(tmpBlock.userKeys) == 0 {
@@ -209,6 +216,73 @@ func mergeBlocks(closeCh <-chan struct{}, bw *blockWriter, br *blockReader) (*pa
return &result, nil
}
+func collectConflictTags(parts []*partWrapper) map[string]struct{} {
+ tagTypes := make(map[string]map[pbv1.ValueType]struct{})
+ for _, pw := range parts {
+ p := pw.p
+ for tt := range p.tagMetadata {
+ t := tt
+ var vt pbv1.ValueType
+ if hasTypeSuffix(tt) {
+ t = decodeTypedTag(tt)
+ vt = valueType(tt)
+ } else {
+ var readErr error
+ vt, readErr = readFirstTagValueType(p.tagMetadata[tt])
+ if readErr != nil {
+ continue
+ }
+ }
+ if tagTypes[t] == nil {
+ tagTypes[t] = make(map[pbv1.ValueType]struct{})
+ }
+ tagTypes[t][vt] = struct{}{}
+ }
+ }
+ var result map[string]struct{}
+ for tag, types := range tagTypes {
+ if len(types) > 1 {
+ if result == nil {
+ result = make(map[string]struct{})
+ }
+ result[tag] = struct{}{}
+ }
+ }
+ return result
+}
+
+func readFirstTagValueType(tmReader fs.Reader) (pbv1.ValueType, error) {
+ sr := tmReader.SequentialRead()
+ defer fs.MustClose(sr)
+ buf := make([]byte, 512)
+ n, readErr := io.ReadFull(sr, buf)
+ if readErr != nil && n == 0 {
+ return pbv1.ValueTypeUnknown, fmt.Errorf("cannot read tag metadata: %w", readErr)
+ }
+ tm := generateTagMetadata()
+ defer releaseTagMetadata(tm)
+ if unmarshalErr := tm.unmarshal(buf[:n]); unmarshalErr != nil {
+ return pbv1.ValueTypeUnknown, fmt.Errorf("cannot unmarshal tag metadata: %w", unmarshalErr)
+ }
+ return tm.valueType, nil
+}
+
+func renameConflictTags(b *block, conflictTags map[string]struct{}) {
+ if len(conflictTags) == 0 {
+ return
+ }
+ for t := range conflictTags {
+ td, exists := b.tags[t]
+ if !exists {
+ continue
+ }
+ typedName := encodeTypedTag(t, td.valueType)
+ td.name = typedName
+ b.tags[typedName] = td
+ delete(b.tags, t)
+ }
+}
+
func mergeTwoBlocks(target, left, right *blockPointer) {
appendIfEmpty := func(ib1, ib2 *blockPointer) bool {
if ib1.idx >= len(ib1.userKeys) {
diff --git a/banyand/internal/sidx/merge_test.go b/banyand/internal/sidx/merge_test.go
index b62f081c0..64c126aa9 100644
--- a/banyand/internal/sidx/merge_test.go
+++ b/banyand/internal/sidx/merge_test.go
@@ -29,6 +29,7 @@ import (
"github.com/apache/skywalking-banyandb/api/common"
"github.com/apache/skywalking-banyandb/banyand/protector"
+ "github.com/apache/skywalking-banyandb/pkg/convert"
"github.com/apache/skywalking-banyandb/pkg/fs"
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/test"
@@ -482,6 +483,126 @@ func Test_mergeParts(t *testing.T) {
}
}
+func Test_mergePartsWithConflictTags(t *testing.T) {
+ esConflictStr := func() *elements {
+ es := generateElements()
+ es.mustAppend(1, 100, make([]byte, 10), []Tag{
+ {Name: "status", Value: []byte("ok"), ValueType: pbv1.ValueTypeStr},
+ })
+ es.mustAppend(2, 200, make([]byte, 10), []Tag{
+ {Name: "status", Value: []byte("error"), ValueType: pbv1.ValueTypeStr},
+ })
+ es.mustAppend(3, 300, make([]byte, 10), []Tag{
+ {Name: "status", Value: []byte("warn"), ValueType: pbv1.ValueTypeStr},
+ })
+ return es
+ }()
+
+ esConflictInt := func() *elements {
+ es := generateElements()
+ es.mustAppend(1, 150, make([]byte, 10), []Tag{
+ {Name: "status", Value: convert.Int64ToBytes(200), ValueType: pbv1.ValueTypeInt64},
+ })
+ es.mustAppend(2, 250, make([]byte, 10), []Tag{
+ {Name: "status", Value: convert.Int64ToBytes(500), ValueType: pbv1.ValueTypeInt64},
+ })
+ es.mustAppend(3, 350, make([]byte, 10), []Tag{
+ {Name: "status", Value: convert.Int64ToBytes(404), ValueType: pbv1.ValueTypeInt64},
+ })
+ return es
+ }()
+
+ verify := func(t *testing.T, pp []*partWrapper, fileSystem fs.FileSystem, root string, partID uint64) {
+ closeCh := make(chan struct{})
+ defer close(closeCh)
+ s := &sidx{pm: protector.Nop{}}
+ p, mergeErr := s.mergeParts(fileSystem, closeCh, pp, partID, root)
+ require.NoError(t, mergeErr)
+ defer func() {
+ if p != nil {
+ p.release()
+ }
+ }()
+
+ pmi := &partMergeIter{}
+ pmi.mustInitFromPart(p.p)
+ reader := &blockReader{}
+ reader.init([]*partMergeIter{pmi})
+ decoder := generateTagValuesDecoder()
+ defer releaseTagValuesDecoder(decoder)
+
+ var blockCount int
+ for reader.nextBlockMetadata() {
+ reader.loadBlockData(decoder)
+ b := &reader.block.block
+ blockCount++
+
+ _, hasBareStatus := b.tags["status"]
+ require.False(t, hasBareStatus,
+ "block %d (seriesID=%d) has bare 'status' tag; expected renamed tags only",
+ blockCount, reader.block.bm.seriesID)
+
+ hasStr := false
+ hasInt := false
+ for tagName := range b.tags {
+ if tagName == "status#str" {
+ hasStr = true
+ }
+ if tagName == "status#int" {
+ hasInt = true
+ }
+ }
+ require.True(t, hasStr || hasInt,
+ "block %d (seriesID=%d) has neither 'status#str' nor 'status#int'",
+ blockCount, reader.block.bm.seriesID)
+ }
+ require.NoError(t, reader.error())
+ require.Equal(t, 3, blockCount, "expected 3 blocks (one per seriesID)")
+ }
+
+ esList := []*elements{esConflictStr, esConflictInt}
+
+ t.Run("memory parts", func(t *testing.T) {
+ var pp []*partWrapper
+ tmpPath, defFn := test.Space(require.New(t))
+ defer func() {
+ for _, pw := range pp {
+ pw.release()
+ }
+ defFn()
+ }()
+ for _, es := range esList {
+ mp := GenerateMemPart()
+ mp.mustInitFromElements(es)
+ pp = append(pp, newPartWrapper(mp, openMemPart(mp)))
+ }
+ verify(t, pp, fs.NewLocalFileSystem(), tmpPath, 1)
+ })
+
+ t.Run("file parts", func(t *testing.T) {
+ var fpp []*partWrapper
+ tmpPath, defFn := test.Space(require.New(t))
+ defer func() {
+ for _, pw := range fpp {
+ pw.release()
+ }
+ defFn()
+ }()
+ fileSystem := fs.NewLocalFileSystem()
+ for i, es := range esList {
+ mp := GenerateMemPart()
+ mp.mustInitFromElements(es)
+ partPath := filepath.Join(tmpPath, "part_"+string(rune('0'+i)))
+ mp.mustFlush(fileSystem, partPath)
+ filePart := mustOpenPart(uint64(i), partPath, fileSystem)
+ filePW := newPartWrapper(nil, filePart)
+ fpp = append(fpp, filePW)
+ ReleaseMemPart(mp)
+ }
+ verify(t, fpp, fileSystem, tmpPath, uint64(len(esList)))
+ })
+}
+
func elementsToBlocks(esList []*elements) []block {
merged := generateElements()
defer releaseElements(merged)
diff --git a/banyand/internal/sidx/query_result.go b/banyand/internal/sidx/query_result.go
index c483058fb..e547d16f0 100644
--- a/banyand/internal/sidx/query_result.go
+++ b/banyand/internal/sidx/query_result.go
@@ -114,16 +114,30 @@ func (qr *queryResult) loadBlockData(tmpBlock *block, p *part, bm *blockMetadata
// Load tag data for selected tags only
for tagName := range tagsToLoad {
tagBlockInfo, exists := bm.tagsBlocks[tagName]
- if !exists {
- continue // Skip missing tags
+ if exists {
+ qr.loadTagData(tmpBlock, p, tagName, &tagBlockInfo, int(bm.count), decoder)
+ continue
}
-
- if !qr.loadTagData(tmpBlock, p, tagName, &tagBlockInfo, int(bm.count), decoder) {
- // Continue loading other tags even if one fails
+ schemaType, hasSchemaType := qr.request.SchemaTagTypes[tagName]
+ if !hasSchemaType {
continue
}
+ for typedTag, typedBlock := range bm.tagsBlocks {
+ if decodeTypedTag(typedTag) != tagName {
+ continue
+ }
+ if valueType(typedTag) != schemaType {
+ continue
+ }
+ if qr.loadTagData(tmpBlock, p, typedTag, &typedBlock, int(bm.count), decoder) {
+ td := tmpBlock.tags[typedTag]
+ td.name = tagName
+ tmpBlock.tags[tagName] = td
+ delete(tmpBlock.tags, typedTag)
+ }
+ break
+ }
}
-
return len(tmpBlock.userKeys) > 0
}
diff --git a/banyand/internal/sidx/scan_query.go b/banyand/internal/sidx/scan_query.go
index 4a063d3a8..c72b3c8f6 100644
--- a/banyand/internal/sidx/scan_query.go
+++ b/banyand/internal/sidx/scan_query.go
@@ -151,16 +151,17 @@ func (s *sidx) scanPart(ctx context.Context, pw *partWrapper, req ScanQueryReque
}
// Create a temporary QueryRequest to reuse existing blockCursor infrastructure
- tmpReq := QueryRequest{
- TagFilter: req.TagFilter,
- MinKey: &minKey,
- MaxKey: &maxKey,
- TagProjection: req.TagProjection,
- MaxBatchSize: maxBatchSize,
+ qr := QueryRequest{
+ TagFilter: req.TagFilter,
+ SchemaTagTypes: req.SchemaTagTypes,
+ MinKey: &minKey,
+ MaxKey: &maxKey,
+ TagProjection: req.TagProjection,
+ MaxBatchSize: maxBatchSize,
}
bc := generateBlockCursor()
- bc.init(p, pi.curBlock, tmpReq)
+ bc.init(p, pi.curBlock, qr)
// Load block data
tmpBlock := generateBlock()
@@ -184,7 +185,7 @@ func (s *sidx) scanPart(ctx context.Context, pw *partWrapper, req ScanQueryReque
if s.loadBlockCursor(bc, tmpBlock, blockScanResult{
p: p,
bm: *pi.curBlock,
- }, tagsToLoad, tmpReq, s.pm, nil) {
+ }, tagsToLoad, qr, s.pm, nil) {
// Copy all rows from this block cursor to current batch
for idx := 0; idx < len(bc.userKeys); idx++ {
bc.idx = idx
diff --git a/banyand/internal/sidx/tag.go b/banyand/internal/sidx/tag.go
index e1f986060..6686f9c88 100644
--- a/banyand/internal/sidx/tag.go
+++ b/banyand/internal/sidx/tag.go
@@ -19,6 +19,7 @@ package sidx
import (
"fmt"
+ "strings"
internalencoding "github.com/apache/skywalking-banyandb/banyand/internal/encoding"
"github.com/apache/skywalking-banyandb/pkg/bytes"
@@ -28,6 +29,66 @@ import (
"github.com/apache/skywalking-banyandb/pkg/pool"
)
+const typedTagSeparator = "#"
+
+var (
+ valueTypeToSuffix = map[pbv1.ValueType]string{
+ pbv1.ValueTypeStr: "str",
+ pbv1.ValueTypeInt64: "int",
+ pbv1.ValueTypeFloat64: "float",
+ pbv1.ValueTypeBinaryData: "bin",
+ pbv1.ValueTypeStrArr: "str_arr",
+ pbv1.ValueTypeInt64Arr: "int_arr",
+ pbv1.ValueTypeTimestamp: "ts",
+ pbv1.ValueTypeUnknown: "",
+ }
+ suffixToValueType = map[string]pbv1.ValueType{
+ "str": pbv1.ValueTypeStr,
+ "int": pbv1.ValueTypeInt64,
+ "float": pbv1.ValueTypeFloat64,
+ "bin": pbv1.ValueTypeBinaryData,
+ "str_arr": pbv1.ValueTypeStrArr,
+ "int_arr": pbv1.ValueTypeInt64Arr,
+ "ts": pbv1.ValueTypeTimestamp,
+ }
+)
+
+func encodeTypedTag(name string, vt pbv1.ValueType) string {
+ suffix, ok := valueTypeToSuffix[vt]
+ if !ok || suffix == "" {
+ return name
+ }
+ return name + typedTagSeparator + suffix
+}
+
+func decodeTypedTag(key string) string {
+ for suffix := range suffixToValueType {
+ sepSuffix := typedTagSeparator + suffix
+ if strings.HasSuffix(key, sepSuffix) {
+ return key[:len(key)-len(sepSuffix)]
+ }
+ }
+ return key
+}
+
+func hasTypeSuffix(key string) bool {
+ for suffix := range suffixToValueType {
+ if strings.HasSuffix(key, typedTagSeparator+suffix) {
+ return true
+ }
+ }
+ return false
+}
+
+func valueType(name string) pbv1.ValueType {
+ for suffix, vt := range suffixToValueType {
+ if strings.HasSuffix(name, typedTagSeparator+suffix) {
+ return vt
+ }
+ }
+ return pbv1.ValueTypeUnknown
+}
+
// dataBlock represents a reference to data in a file.
type dataBlock struct {
offset uint64
@@ -278,18 +339,18 @@ func (tm *tagMetadata) marshal(dst []byte) []byte {
}
// unmarshal deserializes tag metadata from bytes using encoding package.
-func (tm *tagMetadata) unmarshal(src []byte) ([]byte, error) {
+func (tm *tagMetadata) unmarshal(src []byte) error {
var nameBytes []byte
var err error
src, nameBytes, err = pkgencoding.DecodeBytes(src)
if err != nil {
- return nil, fmt.Errorf("cannot unmarshal tagMetadata.name: %w", err)
+ return fmt.Errorf("cannot unmarshal tagMetadata.name: %w", err)
}
tm.name = string(nameBytes)
if len(src) < 1 {
- return nil, fmt.Errorf("cannot unmarshal tagMetadata.valueType: src is too short")
+ return fmt.Errorf("cannot unmarshal tagMetadata.valueType: src is too short")
}
tm.valueType = pbv1.ValueType(src[0])
src = src[1:]
@@ -300,26 +361,26 @@ func (tm *tagMetadata) unmarshal(src []byte) ([]byte, error) {
src, tm.filterBlock.size = pkgencoding.BytesToVarUint64(src)
if len(src) < 1 {
- return nil, fmt.Errorf("cannot unmarshal tagMetadata flags: src is too short")
+ return fmt.Errorf("cannot unmarshal tagMetadata flags: src is too short")
}
src, tm.min, err = pkgencoding.DecodeBytes(src)
if err != nil {
- return nil, fmt.Errorf("cannot unmarshal tagMetadata.min: %w", err)
+ return fmt.Errorf("cannot unmarshal tagMetadata.min: %w", err)
}
if len(tm.min) == 0 {
tm.min = nil
}
- src, tm.max, err = pkgencoding.DecodeBytes(src)
+ _, tm.max, err = pkgencoding.DecodeBytes(src)
if err != nil {
- return nil, fmt.Errorf("cannot unmarshal tagMetadata.max: %w", err)
+ return fmt.Errorf("cannot unmarshal tagMetadata.max: %w", err)
}
if len(tm.max) == 0 {
tm.max = nil
}
- return src, nil
+ return nil
}
// marshalAppend serializes tagMetadata to bytes and appends to dst (panic version for mustWriteTag).
@@ -330,7 +391,7 @@ func (tm *tagMetadata) marshalAppend(dst []byte) []byte {
// unmarshalTagMetadata deserializes tag metadata from bytes.
func unmarshalTagMetadata(data []byte) (*tagMetadata, error) {
tm := generateTagMetadata()
- _, err := tm.unmarshal(data)
+ err := tm.unmarshal(data)
if err != nil {
releaseTagMetadata(tm)
return nil, err
diff --git a/banyand/trace/block.go b/banyand/trace/block.go
index b4e05733b..0db54fdd1 100644
--- a/banyand/trace/block.go
+++ b/banyand/trace/block.go
@@ -465,7 +465,8 @@ func (bc *blockCursor) copyAllTo(r *model.TraceResult) {
}
for i, t := range bc.tags {
values := make([]*modelv1.TagValue, len(bc.spans))
- schemaType, hasSchemaType := bc.schemaTagTypes[t.name]
+ name := decodeTypedTag(t.name)
+ schemaType, hasSchemaType := bc.schemaTagTypes[name]
for k := range bc.spans {
if len(t.values) > k {
if hasSchemaType && t.valueType == schemaType {
@@ -485,18 +486,32 @@ func (bc *blockCursor) loadData(tmpBlock *block) bool {
tmpBlock.reset()
bc.bm.tagProjection = bc.tagProjection
var t map[string]*dataBlock
- for _, name := range bc.tagProjection.Names {
- for tagName, block := range bc.bm.tags {
- if tagName == name {
- if t == nil {
- t = make(map[string]*dataBlock, len(bc.tagProjection.Names))
+ projectionNames := make([]string, len(bc.tagProjection.Names))
+ copy(projectionNames, bc.tagProjection.Names)
+ for i, name := range bc.tagProjection.Names {
+ for typedTag, block := range bc.bm.tags {
+ if decodeTypedTag(typedTag) != name {
+ continue
+ }
+ if hasTypeSuffix(typedTag) {
+ schemaType, ok := bc.schemaTagTypes[name]
+ if !ok {
+ continue
+ }
+ if bmType, ok := bc.bm.tagType[typedTag]; !ok || bmType != schemaType {
+ continue
}
- t[name] = block
+ projectionNames[i] = typedTag
+ }
+ if t == nil {
+ t = make(map[string]*dataBlock, len(bc.tagProjection.Names))
}
+ t[typedTag] = block
}
}
bc.bm.tags = t
+ bc.bm.tagProjection = &model.TagProjection{Names: projectionNames}
tmpBlock.mustReadFrom(&bc.tagValuesDecoder, bc.p, bc.bm)
if len(tmpBlock.spans) == 0 {
return false
@@ -591,6 +606,8 @@ func fastTagAppend(bi, b *blockPointer, offset int) error {
return fmt.Errorf("unexpected tag name for tag %q: got %q; want %q",
bi.tags[i].name, b.tags[i].name, bi.tags[i].name)
}
+ }
+ for i := range bi.tags {
assertIdxAndOffset(b.tags[i].name, len(b.tags[i].values), b.idx, offset)
bi.tags[i].values = append(bi.tags[i].values, b.tags[i].values[b.idx:offset]...)
}
diff --git a/banyand/trace/merger.go b/banyand/trace/merger.go
index d4286fd9d..0ee1c05d1 100644
--- a/banyand/trace/merger.go
+++ b/banyand/trace/merger.go
@@ -30,6 +30,7 @@ import (
"github.com/apache/skywalking-banyandb/pkg/cgroups"
"github.com/apache/skywalking-banyandb/pkg/encoding"
"github.com/apache/skywalking-banyandb/pkg/fs"
+ pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
"github.com/apache/skywalking-banyandb/pkg/watcher"
)
@@ -320,6 +321,7 @@ func (tst *tsTable) mergeParts(fileSystem fs.FileSystem, closeCh <-chan struct{}
br.init(pii)
bw := generateBlockWriter()
bw.mustInitForFilePart(fileSystem, dstPath, shouldCache, int(traceSize))
+ conflictTags := collectConflictTags(parts)
var minTimestamp, maxTimestamp int64
for i, pw := range parts {
@@ -337,7 +339,7 @@ func (tst *tsTable) mergeParts(fileSystem fs.FileSystem, closeCh <-chan struct{}
}
}
- pm, tf, tt, err := mergeBlocks(closeCh, bw, br)
+ pm, tf, tt, err := mergeBlocks(closeCh, bw, br, conflictTags)
releaseBlockWriter(bw)
releaseBlockReader(br)
for i := range pii {
@@ -364,7 +366,30 @@ var errClosed = fmt.Errorf("the merger is closed")
// forceSlowMerge is used for testing to disable the fast raw merge path.
var forceSlowMerge = false
-func mergeBlocks(closeCh <-chan struct{}, bw *blockWriter, br *blockReader) (*partMetadata, *traceIDFilter, *tagType, error) {
+func collectConflictTags(parts []*partWrapper) map[string]struct{} {
+ tagTypes := make(map[string]map[pbv1.ValueType]struct{})
+ for _, pw := range parts {
+ for tag, vt := range pw.p.tagType {
+ t := decodeTypedTag(tag)
+ if tagTypes[t] == nil {
+ tagTypes[t] = make(map[pbv1.ValueType]struct{})
+ }
+ tagTypes[t][vt] = struct{}{}
+ }
+ }
+ var result map[string]struct{}
+ for tag, types := range tagTypes {
+ if len(types) > 1 {
+ if result == nil {
+ result = make(map[string]struct{})
+ }
+ result[tag] = struct{}{}
+ }
+ }
+ return result
+}
+
+func mergeBlocks(closeCh <-chan struct{}, bw *blockWriter, br *blockReader, conflictTags map[string]struct{}) (*partMetadata, *traceIDFilter, *tagType, error) {
pendingBlockIsEmpty := true
pendingBlock := generateBlockPointer()
defer releaseBlockPointer(pendingBlock)
@@ -383,6 +408,14 @@ func mergeBlocks(closeCh <-chan struct{}, bw *blockWriter, br *blockReader) (*pa
decoder = nil
}
}
+ loadAndRename := func() {
+ br.loadBlockData(getDecoder())
+ renameConflictTags(&br.block.block, conflictTags)
+ }
+ readAndRename := func(bm *blockMetadata) {
+ br.mustReadRaw(&rawBlk, bm)
+ renameRawConflictTags(&rawBlk, conflictTags)
+ }
for br.nextBlockMetadata() {
select {
case <-closeCh:
@@ -395,13 +428,13 @@ func mergeBlocks(closeCh <-chan struct{}, bw *blockWriter, br *blockReader) (*pa
nextB := br.peek()
if !forceSlowMerge && pendingBlockIsEmpty && (nextB == nil || nextB.bm.traceID != b.bm.traceID) {
// fast path: only a single block for the trace id and no pending data
- br.mustReadRaw(&rawBlk, &b.bm)
+ readAndRename(&b.bm)
bw.mustWriteRawBlock(&rawBlk)
continue
}
if pendingBlockIsEmpty {
- br.loadBlockData(getDecoder())
+ loadAndRename()
pendingBlock.copyFrom(b)
pendingBlockIsEmpty = false
continue
@@ -416,12 +449,12 @@ func mergeBlocks(closeCh <-chan struct{}, bw *blockWriter, br *blockReader) (*pa
nextB = br.peek()
if !forceSlowMerge && (nextB == nil || nextB.bm.traceID != b.bm.traceID) {
// fast path: only a single block for this new trace id
- br.mustReadRaw(&rawBlk, &b.bm)
+ readAndRename(&b.bm)
bw.mustWriteRawBlock(&rawBlk)
continue
}
// Slow path: start accumulating the new block
- br.loadBlockData(getDecoder())
+ loadAndRename()
pendingBlock.copyFrom(b)
pendingBlockIsEmpty = false
continue
@@ -433,7 +466,7 @@ func mergeBlocks(closeCh <-chan struct{}, bw *blockWriter, br *blockReader) (*pa
}
tmpBlock.reset()
tmpBlock.bm.traceID = b.bm.traceID
- br.loadBlockData(getDecoder())
+ loadAndRename()
mergeTwoBlocks(tmpBlock, pendingBlock, b)
if tmpBlock.block.spanSize() <= maxUncompressedSpanSize {
if len(tmpBlock.spans) == 0 {
@@ -466,3 +499,40 @@ func mergeTwoBlocks(target, left, right *blockPointer) {
target.appendAll(left)
target.appendAll(right)
}
+
+func renameConflictTags(b *block, conflictTags map[string]struct{}) {
+ if len(conflictTags) == 0 {
+ return
+ }
+ for i := range b.tags {
+ if _, ok := conflictTags[b.tags[i].name]; ok {
+ b.tags[i].name = encodeTypedTag(b.tags[i].name, b.tags[i].valueType)
+ }
+ }
+}
+
+func renameRawConflictTags(r *rawBlock, conflictTags map[string]struct{}) {
+ if len(conflictTags) == 0 {
+ return
+ }
+ bm := r.bm
+ for tag := range conflictTags {
+ if _, ok := bm.tags[tag]; !ok {
+ continue
+ }
+ valueType := bm.tagType[tag]
+ typedTag := encodeTypedTag(tag, valueType)
+ bm.tags[typedTag] = bm.tags[tag]
+ delete(bm.tags, tag)
+ bm.tagType[typedTag] = valueType
+ delete(bm.tagType, tag)
+ if rawData, ok := r.tags[tag]; ok {
+ r.tags[typedTag] = rawData
+ delete(r.tags, tag)
+ }
+ if rawMeta, ok := r.tagMetadata[tag]; ok {
+ r.tagMetadata[typedTag] = rawMeta
+ delete(r.tagMetadata, tag)
+ }
+ }
+}
diff --git a/banyand/trace/merger_bench_test.go b/banyand/trace/merger_bench_test.go
index fb2aae7f1..601644c21 100644
--- a/banyand/trace/merger_bench_test.go
+++ b/banyand/trace/merger_bench_test.go
@@ -119,7 +119,7 @@ func Benchmark_mergeBlocks_FastRawMerge(b *testing.B) {
closeCh := make(chan struct{})
- _, _, _, err := mergeBlocks(closeCh, bw, br)
+ _, _, _, err := mergeBlocks(closeCh, bw, br, nil)
if err != nil {
b.Fatal(err)
}
@@ -226,7 +226,7 @@ func Benchmark_mergeBlocks_OriginalMerge(b *testing.B) {
closeCh := make(chan struct{})
- _, _, _, err := mergeBlocks(closeCh, bw, br)
+ _, _, _, err := mergeBlocks(closeCh, bw, br, nil)
if err != nil {
b.Fatal(err)
}
@@ -331,7 +331,7 @@ func Benchmark_mergeBlocks_Comparison(b *testing.B) {
closeCh := make(chan struct{})
- _, _, _, err := mergeBlocks(closeCh, bw, br)
+ _, _, _, err := mergeBlocks(closeCh, bw, br, nil)
if err != nil {
b.Fatal(err)
}
@@ -377,7 +377,7 @@ func Benchmark_mergeBlocks_Comparison(b *testing.B) {
closeCh := make(chan struct{})
- _, _, _, err := mergeBlocks(closeCh, bw, br)
+ _, _, _, err := mergeBlocks(closeCh, bw, br, nil)
if err != nil {
b.Fatal(err)
}
diff --git a/banyand/trace/merger_test.go b/banyand/trace/merger_test.go
index f162b400e..e2c5c4e9f 100644
--- a/banyand/trace/merger_test.go
+++ b/banyand/trace/merger_test.go
@@ -269,7 +269,7 @@ func Test_mergeBlocks_fastPath(t *testing.T) {
closeCh := make(chan struct{})
defer close(closeCh)
- pm, tf, tagTypes, err := mergeBlocks(closeCh, bw, br)
+ pm, tf, tagTypes, err := mergeBlocks(closeCh, bw, br, nil)
require.NoError(t, err)
require.NotNil(t, pm)
require.NotNil(t, tf)
@@ -1456,3 +1456,155 @@ func Test_flush_cleansUpOnSidxFlushError(t *testing.T) {
sidxPartPath := filepath.Join(tmpPath, sidxDirName, "idx1", fmt.Sprintf("%016x", partID))
require.False(t, fileSystem.IsExist(sidxPartPath), "sidx part directory should be removed on sidx flush failure")
}
+
+func Test_mergePartsWithConflictingTagTypes(t *testing.T) {
+ tests := []struct {
+ name string
+ parts []*traces
+ wantTagTypes map[string]pbv1.ValueType
+ wantDeleted []string
+ }{
+ {
+ name: "multiple parts with int and str for same tag",
+ parts: []*traces{
+ {
+ traceIDs: []string{"trace1"},
+ timestamps: []int64{1},
+ tags: [][]*tagValue{
+ {
+ {tag: "state", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(100)},
+ {tag: "name", valueType: pbv1.ValueTypeStr, value: []byte("svc1")},
+ },
+ },
+ spans: [][]byte{[]byte("span1")},
+ spanIDs: []string{"span1"},
+ },
+ {
+ traceIDs: []string{"trace2"},
+ timestamps: []int64{2},
+ tags: [][]*tagValue{
+ {
+ {tag: "state", valueType: pbv1.ValueTypeStr, value: []byte("active")},
+ {tag: "name", valueType: pbv1.ValueTypeStr, value: []byte("svc2")},
+ },
+ },
+ spans: [][]byte{[]byte("span2")},
+ spanIDs: []string{"span2"},
+ },
+ },
+ wantTagTypes: map[string]pbv1.ValueType{
+ "state#int": pbv1.ValueTypeInt64,
+ "state#str": pbv1.ValueTypeStr,
+ "name": pbv1.ValueTypeStr,
+ },
+ wantDeleted: []string{"state"},
+ },
+ {
+ name: "merge typed-suffix part with plain part",
+ parts: []*traces{
+ {
+ traceIDs: []string{"trace1"},
+ timestamps: []int64{1},
+ tags: [][]*tagValue{
+ {{tag: "state#int", valueType: pbv1.ValueTypeInt64, value: convert.Int64ToBytes(100)}},
+ },
+ spans: [][]byte{[]byte("span1")},
+ spanIDs: []string{"span1"},
+ },
+ {
+ traceIDs: []string{"trace2"},
+ timestamps: []int64{2},
+ tags: [][]*tagValue{
+ {{tag: "state", valueType: pbv1.ValueTypeStr, value: []byte("pending")}},
+ },
+ spans: [][]byte{[]byte("span2")},
+ spanIDs: []string{"span2"},
+ },
+ },
+ wantTagTypes: map[string]pbv1.ValueType{
+ "state#int": pbv1.ValueTypeInt64,
+ "state#str": pbv1.ValueTypeStr,
+ },
+ wantDeleted: []string{"state"},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ tmpPath, defFn := test.Space(require.New(t))
+ defer defFn()
+ fileSystem := fs.NewLocalFileSystem()
+
+ partIDs := make([]uint64, len(tt.parts))
+ for idx := range partIDs {
+ partIDs[idx] = uint64(idx)
+ }
+ dstID := uint64(999)
+ dstPath := mergePartsWithConflictTags(t, fileSystem, tmpPath, tt.parts, partIDs, dstID)
+
+ mergedPart := mustOpenFilePart(dstID, tmpPath, fileSystem)
+ for tag, wantType := range tt.wantTagTypes {
+ gotType, exists := mergedPart.tagType[tag]
+ require.True(t, exists, "expected tag type entry for %q", tag)
+ require.Equal(t, wantType, gotType, "tag type mismatch for %q", tag)
+ }
+ for _, tag := range tt.wantDeleted {
+ _, ok := mergedPart.tagType[tag]
+ require.False(t, ok, "tag %q should not exist in tagType after merge", tag)
+ dataFile := filepath.Join(dstPath, tag+tagsFilenameExt)
+ require.False(t, fileSystem.IsExist(dataFile), "tag data file %q should not exist", dataFile)
+ metaFile := filepath.Join(dstPath, tag+tagsMetadataFilenameExt)
+ require.False(t, fileSystem.IsExist(metaFile), "tag metadata file %q should not exist", metaFile)
+ }
+ })
+ }
+}
+
+func mergePartsWithConflictTags(t *testing.T, fileSystem fs.FileSystem, tmpPath string, tsList []*traces, partIDs []uint64, dstID uint64) string {
+ t.Helper()
+ var parts []*partWrapper
+ for i, traces := range tsList {
+ mp := generateMemPart()
+ mp.mustInitFromTraces(traces)
+ mp.mustFlush(fileSystem, partPath(tmpPath, partIDs[i]))
+ p := mustOpenFilePart(partIDs[i], tmpPath, fileSystem)
+ parts = append(parts, &partWrapper{p: p})
+ releaseMemPart(mp)
+ }
+
+ conflictTags := collectConflictTags(parts)
+
+ var pmi []*partMergeIter
+ var traceSize uint64
+ for i := range tsList {
+ p := mustOpenFilePart(partIDs[i], tmpPath, fileSystem)
+ iter := generatePartMergeIter()
+ iter.mustInitFromPart(p)
+ pmi = append(pmi, iter)
+ traceSize += uint64(len(tsList[i].traceIDs))
+ }
+
+ br := generateBlockReader()
+ br.init(pmi)
+ bw := generateBlockWriter()
+ dstPath := partPath(tmpPath, dstID)
+ bw.mustInitForFilePart(fileSystem, dstPath, false, int(traceSize))
+
+ closeCh := make(chan struct{})
+ pm, tf, tagTypes, mergeErr := mergeBlocks(closeCh, bw, br, conflictTags)
+ close(closeCh)
+ require.NoError(t, mergeErr)
+ require.NotNil(t, pm)
+
+ releaseBlockWriter(bw)
+ releaseBlockReader(br)
+ for _, iter := range pmi {
+ releasePartMergeIter(iter)
+ }
+
+ pm.mustWriteMetadata(fileSystem, dstPath)
+ tf.mustWriteTraceIDFilter(fileSystem, dstPath)
+ tagTypes.mustWriteTagType(fileSystem, dstPath)
+ fileSystem.SyncPath(dstPath)
+ return dstPath
+}
diff --git a/banyand/trace/query.go b/banyand/trace/query.go
index 801b70872..7845d0eb7 100644
--- a/banyand/trace/query.go
+++ b/banyand/trace/query.go
@@ -252,13 +252,14 @@ func (t *trace) prepareSIDXStreaming(
}
req := sidx.QueryRequest{
- Filter: tqo.SkippingFilter,
- TagFilter: tqo.TagFilter,
- Order: tqo.Order,
- MaxBatchSize: tqo.MaxTraceSize,
- MinKey: &tqo.MinVal,
- MaxKey: &tqo.MaxVal,
- SeriesIDs: seriesIDs,
+ Filter: tqo.SkippingFilter,
+ TagFilter: tqo.TagFilter,
+ Order: tqo.Order,
+ MaxBatchSize: tqo.MaxTraceSize,
+ MinKey: &tqo.MinVal,
+ MaxKey: &tqo.MaxVal,
+ SeriesIDs: seriesIDs,
+ SchemaTagTypes: qo.schemaTagTypes,
}
if tqo.TagProjection != nil {
req.TagProjection = []model.TagProjection{*tqo.TagProjection}
diff --git a/banyand/trace/svc_standalone.go b/banyand/trace/svc_standalone.go
index 77dd2f873..13f4e94e4 100644
--- a/banyand/trace/svc_standalone.go
+++ b/banyand/trace/svc_standalone.go
@@ -95,6 +95,7 @@ func (s *standalone) FlagSet() *run.FlagSet {
fs.IntVar(&s.maxFileSnapshotNum, "trace-max-file-snapshot-num", 10, "the maximum number of file snapshots")
fs.DurationVar(&s.minFileSnapshotAge, "trace-min-file-snapshot-age", time.Hour, "minimum age for file snapshots to be eligible for deletion")
s.option.mergePolicy = newDefaultMergePolicy()
+ fs.IntVar(&s.option.mergePolicy.maxParts, "trace-max-merge-parts", s.option.mergePolicy.maxParts, "the maximum number of parts to merge at once")
fs.VarP(&s.option.mergePolicy.maxFanOutSize, "trace-max-fan-out-size", "", "the upper bound of a single file size after merge of trace")
// Additional flags can be added here
return fs
diff --git a/banyand/trace/tag.go b/banyand/trace/tag.go
index 647358da7..31c612f9b 100644
--- a/banyand/trace/tag.go
+++ b/banyand/trace/tag.go
@@ -18,6 +18,8 @@
package trace
import (
+ "strings"
+
internalencoding "github.com/apache/skywalking-banyandb/banyand/internal/encoding"
"github.com/apache/skywalking-banyandb/pkg/bytes"
pkgencoding "github.com/apache/skywalking-banyandb/pkg/encoding"
@@ -26,6 +28,57 @@ import (
pbv1 "github.com/apache/skywalking-banyandb/pkg/pb/v1"
)
+const typedTagSeparator = "#"
+
+var (
+ valueTypeToSuffix = map[pbv1.ValueType]string{
+ pbv1.ValueTypeStr: "str",
+ pbv1.ValueTypeInt64: "int",
+ pbv1.ValueTypeFloat64: "float",
+ pbv1.ValueTypeBinaryData: "bin",
+ pbv1.ValueTypeStrArr: "str_arr",
+ pbv1.ValueTypeInt64Arr: "int_arr",
+ pbv1.ValueTypeTimestamp: "ts",
+ pbv1.ValueTypeUnknown: "",
+ }
+ suffixToValueType = map[string]pbv1.ValueType{
+ "str": pbv1.ValueTypeStr,
+ "int": pbv1.ValueTypeInt64,
+ "float": pbv1.ValueTypeFloat64,
+ "bin": pbv1.ValueTypeBinaryData,
+ "str_arr": pbv1.ValueTypeStrArr,
+ "int_arr": pbv1.ValueTypeInt64Arr,
+ "ts": pbv1.ValueTypeTimestamp,
+ }
+)
+
+func encodeTypedTag(name string, vt pbv1.ValueType) string {
+ suffix, ok := valueTypeToSuffix[vt]
+ if !ok || suffix == "" {
+ return name
+ }
+ return name + typedTagSeparator + suffix
+}
+
+func decodeTypedTag(key string) string {
+ for suffix := range suffixToValueType {
+ sepSuffix := typedTagSeparator + suffix
+ if strings.HasSuffix(key, sepSuffix) {
+ return key[:len(key)-len(sepSuffix)]
+ }
+ }
+ return key
+}
+
+func hasTypeSuffix(key string) bool {
+ for suffix := range suffixToValueType {
+ if strings.HasSuffix(key, typedTagSeparator+suffix) {
+ return true
+ }
+ }
+ return false
+}
+
type tag struct {
name string
values [][]byte
diff --git a/banyand/trace/tag_test.go b/banyand/trace/tag_test.go
index 2ac463d74..fce0a0233 100644
--- a/banyand/trace/tag_test.go
+++ b/banyand/trace/tag_test.go
@@ -34,6 +34,25 @@ func timeToBytes(t time.Time) []byte {
return pkgencoding.Int64ToBytes(nil, t.UnixNano())
}
+func TestEncodeTypedTag(t *testing.T) {
+ assert.Equal(t, "state#int", encodeTypedTag("state", pbv1.ValueTypeInt64))
+ assert.Equal(t, "state#str", encodeTypedTag("state", pbv1.ValueTypeStr))
+ assert.Equal(t, "http.status#int", encodeTypedTag("http.status", pbv1.ValueTypeInt64))
+ assert.Equal(t, "data#bin", encodeTypedTag("data", pbv1.ValueTypeBinaryData))
+ assert.Equal(t, "unknown", encodeTypedTag("unknown", pbv1.ValueTypeUnknown))
+}
+
+func TestDecodeTypedTag(t *testing.T) {
+ assert.Equal(t, "state", decodeTypedTag("state#int"))
+ assert.Equal(t, "state", decodeTypedTag("state#str"))
+ assert.Equal(t, "http.status", decodeTypedTag("http.status#int"))
+ assert.Equal(t, "http.status", decodeTypedTag("http.status"))
+ assert.Equal(t, "simple", decodeTypedTag("simple"))
+ assert.Equal(t, "my.str", decodeTypedTag("my.str"))
+ assert.Equal(t, "arr.int", decodeTypedTag("arr.int"))
+ assert.Equal(t, "data.bin", decodeTypedTag("data.bin"))
+}
+
func TestTagEncodingDecoding(t *testing.T) {
t.Run("test int64 tag encoding/decoding", func(t *testing.T) {
tag := &tag{
diff --git a/banyand/trace/trace_suite_test.go b/banyand/trace/trace_suite_test.go
index daefee2bd..9e69e4790 100644
--- a/banyand/trace/trace_suite_test.go
+++ b/banyand/trace/trace_suite_test.go
@@ -90,7 +90,7 @@ func setUp() (*services, func()) {
flags = append(flags, "--metadata-root-path="+metaPath)
rootPath, deferFunc, err := test.NewSpace()
gomega.Expect(err).NotTo(gomega.HaveOccurred())
- flags = append(flags, "--trace-root-path="+rootPath)
+ flags = append(flags, "--trace-root-path="+rootPath, "--trace-max-merge-parts=2")
listenClientURL, listenPeerURL, err := test.NewEtcdListenUrls()
gomega.Expect(err).NotTo(gomega.HaveOccurred())
flags = append(flags, "--etcd-listen-client-url="+listenClientURL, "--etcd-listen-peer-url="+listenPeerURL,
(skywalking-banyandb) 06/14: fix(trace): resolve flaky schema change merge test (#1099)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 628f5def18ee59948edb02370b272faabfd85edd
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Tue Apr 28 19:31:28 2026 +0800
fix(trace): resolve flaky schema change merge test (#1099)
* docs: add snapshot retention flags and relative path version note
Document the min-file-snapshot-age flags for measure, stream, trace,
and property services. Add version annotation for relative path support.
---
api/proto/banyandb/database/v1/rpc.proto | 2 +
banyand/measure/metadata.go | 3 +-
banyand/stream/metadata.go | 4 +-
banyand/trace/metadata.go | 4 +-
banyand/trace/metadata_test.go | 81 ++++++++++++++++++++++++++++++++
docs/api-reference.md | 1 +
pkg/timestamp/scheduler.go | 8 +++-
7 files changed, 98 insertions(+), 5 deletions(-)
diff --git a/api/proto/banyandb/database/v1/rpc.proto b/api/proto/banyandb/database/v1/rpc.proto
index d9a1cda92..f877722fa 100644
--- a/api/proto/banyandb/database/v1/rpc.proto
+++ b/api/proto/banyandb/database/v1/rpc.proto
@@ -515,6 +515,8 @@ message ShardInfo {
InvertedIndexInfo inverted_index_info = 5;
// sidx_info contains information about sidx.
SIDXInfo sidx_info = 6;
+ // file_part_count is the number of file parts (excluding in-memory parts) in this shard.
+ int64 file_part_count = 7;
}
// SeriesIndexInfo contains information about the series index.
diff --git a/banyand/measure/metadata.go b/banyand/measure/metadata.go
index d846f5f0d..d206cd342 100644
--- a/banyand/measure/metadata.go
+++ b/banyand/measure/metadata.go
@@ -455,12 +455,13 @@ func (sr *schemaRepo) collectShardInfo(table any, shardID uint32) *databasev1.Sh
}
}
defer snapshot.decRef()
- var totalCount, compressedSize, partCount uint64
+ var totalCount, compressedSize, partCount, filePartCount uint64
for _, pw := range snapshot.parts {
if pw.p != nil {
totalCount += pw.p.partMetadata.TotalCount
compressedSize += pw.p.partMetadata.CompressedSizeBytes
partCount++
+ filePartCount++
} else if pw.mp != nil {
totalCount += pw.mp.partMetadata.TotalCount
compressedSize += pw.mp.partMetadata.CompressedSizeBytes
diff --git a/banyand/stream/metadata.go b/banyand/stream/metadata.go
index 96c810213..dd91f1481 100644
--- a/banyand/stream/metadata.go
+++ b/banyand/stream/metadata.go
@@ -344,13 +344,14 @@ func (sr *schemaRepo) collectShardInfo(table any, shardID uint32) *databasev1.Sh
}
}
defer snapshot.decRef()
- var totalCount, compressedSize, uncompressedSize, partCount uint64
+ var totalCount, compressedSize, uncompressedSize, partCount, filePartCount uint64
for _, pw := range snapshot.parts {
if pw.p != nil {
totalCount += pw.p.partMetadata.TotalCount
compressedSize += pw.p.partMetadata.CompressedSizeBytes
uncompressedSize += pw.p.partMetadata.UncompressedSizeBytes
partCount++
+ filePartCount++
} else if pw.mp != nil {
totalCount += pw.mp.partMetadata.TotalCount
compressedSize += pw.mp.partMetadata.CompressedSizeBytes
@@ -366,6 +367,7 @@ func (sr *schemaRepo) collectShardInfo(table any, shardID uint32) *databasev1.Sh
PartCount: int64(partCount),
InvertedIndexInfo: invertedIndexInfo,
SidxInfo: &databasev1.SIDXInfo{},
+ FilePartCount: int64(filePartCount),
}
}
diff --git a/banyand/trace/metadata.go b/banyand/trace/metadata.go
index 5bbdfe174..3b02231b5 100644
--- a/banyand/trace/metadata.go
+++ b/banyand/trace/metadata.go
@@ -353,13 +353,14 @@ func (sr *schemaRepo) collectShardInfo(ctx context.Context, table any, shardID u
}
}
defer snapshot.decRef()
- var totalCount, compressedSize, uncompressedSize, partCount uint64
+ var totalCount, compressedSize, uncompressedSize, partCount, filePartCount uint64
for _, pw := range snapshot.parts {
if pw.p != nil {
totalCount += pw.p.partMetadata.TotalCount
compressedSize += pw.p.partMetadata.CompressedSizeBytes
uncompressedSize += pw.p.partMetadata.UncompressedSpanSizeBytes
partCount++
+ filePartCount++
} else if pw.mp != nil {
totalCount += pw.mp.partMetadata.TotalCount
compressedSize += pw.mp.partMetadata.CompressedSizeBytes
@@ -375,6 +376,7 @@ func (sr *schemaRepo) collectShardInfo(ctx context.Context, table any, shardID u
PartCount: int64(partCount),
InvertedIndexInfo: &databasev1.InvertedIndexInfo{},
SidxInfo: sidxInfo,
+ FilePartCount: int64(filePartCount),
}
}
diff --git a/banyand/trace/metadata_test.go b/banyand/trace/metadata_test.go
index 8cad466f2..13c74a30e 100644
--- a/banyand/trace/metadata_test.go
+++ b/banyand/trace/metadata_test.go
@@ -358,6 +358,58 @@ var _ = Describe("Metadata", func() {
})
})
+ Context("Trace schema with changed tag type after merge", func() {
+ It("querying data should return correct values after parts with different types are merged", func() {
+ traceName := "schema_change_tag_type_merge"
+ now := timestamp.NowMilli()
+
+ env := setupSchemaChangeTrace(svcs, traceName, groupName, traceSetupOptions{withExtraTag: true})
+ writeSchemaChangeTraceData(svcs, traceName, groupName, now.Add(-2*time.Hour), 5,
+ writeTraceDataOptions{extraTag: extraTagInt})
+ // Wait for the first batch to flush to disk before writing the second batch.
+ // This ensures each shard has at least one file part from the first batch,
+ // so the second batch creates additional parts that can be merged.
+ Eventually(func() int64 {
+ return getFilePartCount(svcs, groupName)
+ }, flags.EventuallyTimeout).Should(BeNumerically(">=", 1))
+ changeTraceExtraTagType(svcs, traceName, groupName)
+ writeSchemaChangeTraceData(svcs, traceName, groupName, now.Add(-1*time.Hour), 3,
+ writeTraceDataOptions{extraTag: extraTagString, traceIDPrefix: "trace_new_"})
+ partCountBeforeMerge := getTotalPartCount(svcs, groupName)
+ Eventually(func() int64 {
+ return getTotalPartCount(svcs, groupName)
+ }, flags.EventuallyTimeout).Should(BeNumerically("<", partCountBeforeMerge))
+
+ Eventually(func(innerGm Gomega) {
+ spans := querySchemaChangeTraceData(svcs, traceName, groupName,
+ now.Add(-3*time.Hour), now,
+ []string{"trace_id", "service_id", "duration", "extra_tag"}, nil)
+ innerGm.Expect(spans).To(HaveLen(8))
+
+ nullCount := 0
+ stringCount := 0
+ for _, span := range spans {
+ for _, tag := range span.Tags {
+ if tag.Key == "extra_tag" {
+ switch tag.Value.GetValue().(type) {
+ case *modelv1.TagValue_Null:
+ nullCount++
+ case *modelv1.TagValue_Str:
+ stringCount++
+ }
+ }
+ }
+ }
+ innerGm.Expect(nullCount).To(Equal(5),
+ "old data with int type should return null after schema changed to STRING and parts merged")
+ innerGm.Expect(stringCount).To(Equal(3),
+ "new data should have string extra_tag values after merge")
+ }, flags.EventuallyTimeout).Should(Succeed())
+
+ env.cleanup()
+ })
+ })
+
Context("Trace schema with deleted tag in query", func() {
It("querying data should fail if the condition includes a deleted tag", func() {
traceName := "schema_change_filter_deleted"
@@ -694,3 +746,32 @@ func querySchemaChangeTraceData(svcs *services, name, group string, begin, end t
}).WithTimeout(flags.EventuallyTimeout).Should(BeTrue())
return spans
}
+
+func getTotalPartCount(svcs *services, group string) int64 {
+ dataInfo, err := svcs.trace.CollectDataInfo(context.TODO(), group)
+ if err != nil || dataInfo == nil {
+ return 0
+ }
+ var total int64
+ for _, seg := range dataInfo.SegmentInfo {
+ for _, shard := range seg.ShardInfo {
+ total += shard.PartCount
+ }
+ }
+ return total
+}
+
+func getFilePartCount(svcs *services, group string) int64 {
+ dataInfo, err := svcs.trace.CollectDataInfo(context.TODO(), group)
+ if err != nil || dataInfo == nil {
+ return 0
+ }
+ var total int64
+ for _, seg := range dataInfo.SegmentInfo {
+ for _, shard := range seg.ShardInfo {
+ total += shard.FilePartCount
+ }
+ }
+ return total
+}
+
diff --git a/docs/api-reference.md b/docs/api-reference.md
index 44696cff7..4a5bc34e9 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -3958,6 +3958,7 @@ ShardInfo contains information about a specific shard.
| part_count | [int64](#int64) | | part_count is the number of parts in this shard. |
| inverted_index_info | [InvertedIndexInfo](#banyandb-database-v1-InvertedIndexInfo) | | inverted_index_info contains information about the inverted index. |
| sidx_info | [SIDXInfo](#banyandb-database-v1-SIDXInfo) | | sidx_info contains information about sidx. |
+| file_part_count | [int64](#int64) | | file_part_count is the number of file parts (excluding in-memory parts) in this shard. |
diff --git a/pkg/timestamp/scheduler.go b/pkg/timestamp/scheduler.go
index b98df0b01..7ee533e2e 100644
--- a/pkg/timestamp/scheduler.go
+++ b/pkg/timestamp/scheduler.go
@@ -150,12 +150,16 @@ func (s *Scheduler) Closed() bool {
// Close the Scheduler and shut down all registered tasks.
func (s *Scheduler) Close() {
s.Lock()
- defer s.Unlock()
s.closed = true
+ tasks := make([]*task, 0, len(s.tasks))
for k, t := range s.tasks {
- t.close()
+ tasks = append(tasks, t)
delete(s.tasks, k)
}
+ s.Unlock()
+ for _, t := range tasks {
+ t.close()
+ }
}
// Metrics returns the metrics of all registered tasks.
(skywalking-banyandb) 10/14: Fix fetch segment nil pointer in cold stage and propagation the error when inspect the group (#1115)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 6ce81416252e9b73f3da7e0c4e896aee408acc6a
Author: mrproliu <74...@qq.com>
AuthorDate: Thu May 7 11:23:26 2026 +0800
Fix fetch segment nil pointer in cold stage and propagation the error when inspect the group (#1115)
---
CHANGES.md | 2 +
api/proto/banyandb/fodc/v1/rpc.proto | 11 +
banyand/internal/storage/index.go | 49 +++-
banyand/internal/storage/segment.go | 42 ++-
banyand/internal/storage/segment_test.go | 318 +++++++++++++++++++++
banyand/liaison/grpc/deletion.go | 14 +-
banyand/liaison/grpc/deletion_test.go | 28 +-
banyand/liaison/grpc/registry.go | 9 +-
banyand/measure/metadata_internal_test.go | 48 ++++
banyand/metadata/client.go | 2 +-
banyand/metadata/metadata.go | 2 +-
banyand/metadata/schema/collector.go | 46 ++-
banyand/queue/sub/group_lifecycle.go | 14 +-
.../queue/sub/group_lifecycle_cold_tier_test.go | 298 +++++++++++++++++++
banyand/queue/sub/group_lifecycle_test.go | 116 +++++++-
banyand/stream/metadata_internal_test.go | 48 ++++
banyand/trace/metadata_internal_test.go | 48 ++++
docs/api-reference.md | 1 +
fodc/proxy/internal/lifecycle/manager.go | 28 +-
fodc/proxy/internal/lifecycle/manager_test.go | 61 ++++
20 files changed, 1146 insertions(+), 39 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index c4b2c5cb7..5cd522b15 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -32,6 +32,8 @@ Release Notes.
- Fix FODC lifecycle cache poisoning where transient `InspectAll` failures were cached for 10 minutes and masked liaison recovery; raise FODC agent and proxy timeouts from 10s to 40s.
- Fix FODC `/cluster/lifecycle` dropping zero-valued group fields (e.g. `replicas=0`, `close=false`) under `encoding/json` + `omitempty`; switch to `protojson` so all fields are emitted (nil nested messages serialize as `null`).
- Fix trace `block_writer` panic on out-of-order timestamps within the same traceID, which dropped one trace-write batch per panic in multi-agent SkyWalking deployments. Spans of a single trace originate from independently-clocked services, and trace storage is organized by traceID rather than timestamp, so per-traceID timestamp monotonicity is not a writer invariant.
+- Fix nil-pointer panic on cold-tier data nodes when FODC `InspectAll` raced with idle-segment cleanup.
+- Add `GroupLifecycleInfo.errors` to surface per-group collection failures from FODC `InspectAll` instead of silently dropping the affected node entry.
### Chores
diff --git a/api/proto/banyandb/fodc/v1/rpc.proto b/api/proto/banyandb/fodc/v1/rpc.proto
index dd7a5c67a..9315da8a6 100644
--- a/api/proto/banyandb/fodc/v1/rpc.proto
+++ b/api/proto/banyandb/fodc/v1/rpc.proto
@@ -114,6 +114,17 @@ message GroupLifecycleInfo {
string catalog = 2;
banyandb.common.v1.ResourceOpts resource_opts = 3;
repeated banyandb.database.v1.DataInfo data_info = 4;
+ // errors lists every failure observed while collecting data_info for this
+ // group: top-level CollectDataInfo failures (GetGroup, missing collector,
+ // dial failure -- prefixed "top-level: ") and per-node broadcast failures
+ // (prefixed "future error: ", "node error: ", "broadcast failed: ").
+ // Combined with len(data_info), consumers can tell the following four
+ // states apart:
+ // - data_info empty && errors empty -> no nodes reported (group inactive)
+ // - data_info empty && errors non-empty -> total failure
+ // - data_info non-empty && errors empty -> full success
+ // - data_info non-empty && errors non-empty -> partial failure
+ repeated string errors = 5;
}
message LifecycleReport {
diff --git a/banyand/internal/storage/index.go b/banyand/internal/storage/index.go
index dff32d742..3ff0ae41c 100644
--- a/banyand/internal/storage/index.go
+++ b/banyand/internal/storage/index.go
@@ -37,14 +37,33 @@ import (
const seriesIndexDirName = "sidx"
func (s *segment[T, O]) IndexDB() IndexDB {
- return s.index
+ // Snapshot s.index into a local: the field can be flipped to nil by
+ // performCleanup at any time, so a second read could see a different
+ // value. Returning the local also avoids boxing a nil *seriesIndex
+ // into a typed-nil IndexDB interface, which would defeat the
+ // `if indexDB == nil` guard on the caller side.
+ idx := s.index
+ if idx == nil {
+ return nil
+ }
+ return idx
}
func (s *segment[T, O]) Lookup(ctx context.Context, series []*pbv1.Series) (pbv1.SeriesList, error) {
- sl, err := s.index.filter(ctx, series, nil, nil, nil)
+ idx := s.index
+ if idx == nil {
+ return nil, ErrSegmentClosed
+ }
+ sl, err := idx.filter(ctx, series, nil, nil, nil)
return sl.SeriesList, err
}
+// seriesIndex is the per-segment series index. Every method on this type
+// starts with `if s == nil { ... }` purely to absorb the typed-nil leak
+// that `segment.IndexDB()` could expose if a caller bypassed the producer
+// guard there. The guards intentionally do NOT make the type fully
+// nil-safe -- they assume that on a non-nil receiver, store / l / metrics
+// / p are all populated by newSeriesIndex (the only constructor).
type seriesIndex struct {
store index.SeriesStore
l *logger.Logger
@@ -79,28 +98,45 @@ func newSeriesIndex(ctx context.Context, root string, flushTimeoutSeconds int64,
}
func (s *seriesIndex) Insert(docs index.Documents) error {
+ if s == nil {
+ return ErrSegmentClosed
+ }
return s.store.InsertSeriesBatch(index.Batch{
Documents: docs,
})
}
func (s *seriesIndex) Update(docs index.Documents) error {
+ if s == nil {
+ return ErrSegmentClosed
+ }
return s.store.UpdateSeriesBatch(index.Batch{
Documents: docs,
})
}
func (s *seriesIndex) EnableExternalSegments() (index.ExternalSegmentStreamer, error) {
+ if s == nil {
+ return nil, ErrSegmentClosed
+ }
return s.store.EnableExternalSegments()
}
+// Stats degrades to (0, 0) on a nil receiver as defense-in-depth in case
+// a typed-nil interface leaks past segment.IndexDB().
func (s *seriesIndex) Stats() (dataCount int64, dataSizeBytes int64) {
+ if s == nil {
+ return 0, 0
+ }
return s.store.Stats()
}
func (s *seriesIndex) filter(ctx context.Context, series []*pbv1.Series,
projection []index.FieldKey, secondaryQuery index.Query, timeRange *timestamp.TimeRange,
) (data SeriesData, err error) {
+ if s == nil {
+ return SeriesData{}, ErrSegmentClosed
+ }
if len(series) == 0 && secondaryQuery == nil && timeRange == nil {
return data, nil
}
@@ -231,6 +267,9 @@ func convertIndexSeriesToSeriesList(indexSeries []index.SeriesDocument, hasField
func (s *seriesIndex) Search(ctx context.Context, series []*pbv1.Series, opts IndexSearchOpts,
) (sd SeriesData, sortedValues [][]byte, err error) {
+ if s == nil {
+ return SeriesData{}, nil, ErrSegmentClosed
+ }
tracer := query.GetTracer(ctx)
if tracer != nil {
var span *query.Span
@@ -309,6 +348,9 @@ func (s *seriesIndex) Search(ctx context.Context, series []*pbv1.Series, opts In
}
func (s *seriesIndex) SearchWithoutSeries(ctx context.Context, opts IndexSearchOpts) (sd SeriesData, sortedValues [][]byte, err error) {
+ if s == nil {
+ return SeriesData{}, nil, ErrSegmentClosed
+ }
tracer := query.GetTracer(ctx)
if tracer != nil {
var span *query.Span
@@ -384,6 +426,9 @@ func (s *seriesIndex) SearchWithoutSeries(ctx context.Context, opts IndexSearchO
}
func (s *seriesIndex) Close() error {
+ if s == nil {
+ return nil
+ }
s.metrics.DeleteAll(s.p.SegLabelValues()...)
return s.store.Close()
}
diff --git a/banyand/internal/storage/segment.go b/banyand/internal/storage/segment.go
index 381a11478..12cc0fed9 100644
--- a/banyand/internal/storage/segment.go
+++ b/banyand/internal/storage/segment.go
@@ -157,13 +157,41 @@ func (s *segment[T, O]) TablesWithShardIDs() (tt []T, shardIDs []common.ShardID,
return tt, shardIDs, cc
}
+// incRef acquires one reference on the segment, opening it via initialize
+// if it is currently closed (refCount<=0). On return, the caller is
+// guaranteed that the segment is alive (s.index != nil, shards loaded) and
+// will stay alive until the matching DecRef.
+//
+// The CAS loop closes a TOCTOU race that the older "Load + AddInt32"
+// shape suffered from: a goroutine could read refCount=1, race against a
+// concurrent DecRef that drives it to 0 and triggers performCleanup
+// (which sets s.index=nil and tears down shards), and then AddInt32 it
+// back to 1. The caller would walk away with refCount=1 -- formally a
+// valid reference -- but pointing at an already-cleaned-up segment, so
+// every subsequent IndexDB() / Tables() call would observe zero state.
+//
+// The CAS variant retries whenever a concurrent DecRef beats us to the
+// counter: if CAS fails, we re-Load and either succeed at the new value
+// or fall through to initialize() to reopen the segment under s.mu.
+// Symmetric to the CAS loop in DecRef, which guarantees the dual: a
+// DecRef that drives refCount to 0 sees no concurrent +1 sneak in
+// before performCleanup runs.
func (s *segment[T, O]) incRef(ctx context.Context) error {
s.lastAccessed.Store(time.Now().UnixNano())
- if atomic.LoadInt32(&s.refCount) <= 0 {
- return s.initialize(ctx)
+ for {
+ current := atomic.LoadInt32(&s.refCount)
+ if current <= 0 {
+ // Either the segment was never opened or DecRef just drove
+ // refCount to 0; reopen under the mutex.
+ return s.initialize(ctx)
+ }
+ // CAS so a concurrent DecRef cannot flip refCount to 0 and run
+ // performCleanup between our Load and our increment. On failure
+ // we re-Load and re-evaluate the branch.
+ if atomic.CompareAndSwapInt32(&s.refCount, current, current+1) {
+ return nil
+ }
}
- atomic.AddInt32(&s.refCount, 1)
- return nil
}
func (s *segment[T, O]) initialize(ctx context.Context) error {
@@ -171,6 +199,12 @@ func (s *segment[T, O]) initialize(ctx context.Context) error {
defer s.mu.Unlock()
if atomic.LoadInt32(&s.refCount) > 0 {
+ // Another goroutine reopened the segment while we waited for the
+ // mutex. Add the +1 our caller (incRef) skipped before entering
+ // the slow path; otherwise concurrent reopens would share one
+ // refCount and the first DecRef would cleanup while we are still
+ // using it.
+ atomic.AddInt32(&s.refCount, 1)
return nil
}
diff --git a/banyand/internal/storage/segment_test.go b/banyand/internal/storage/segment_test.go
index 9a55dd358..f50e07603 100644
--- a/banyand/internal/storage/segment_test.go
+++ b/banyand/internal/storage/segment_test.go
@@ -24,6 +24,8 @@ import (
"os"
"path/filepath"
"strconv"
+ "sync"
+ "sync/atomic"
"testing"
"time"
@@ -860,3 +862,319 @@ func TestOpenFallbackOldFormatMetadata(t *testing.T) {
seg.DecRef()
}
}
+
+// TestSegment_IndexDB_ReturnsUntypedNilWhenClosed is a regression guard for
+// the cold-tier nil-pointer panic. After a segment has been idle-closed
+// (s.index == nil), segment.IndexDB() must return an untyped nil interface
+// so that the standard caller pattern `if indexDB == nil { ... }` short
+// circuits correctly. Returning the underlying *seriesIndex would box a
+// typed-nil interface, defeat the nil check, and crash on the next method
+// call.
+func TestSegment_IndexDB_ReturnsUntypedNilWhenClosed(t *testing.T) {
+ tempDir, cleanup := setupTestEnvironment(t)
+ defer cleanup()
+
+ ctx := context.Background()
+ l := logger.GetLogger("test-untyped-nil")
+ ctx = context.WithValue(ctx, logger.ContextKey, l)
+ ctx = common.SetPosition(ctx, func(_ common.Position) common.Position {
+ return common.Position{Database: "test-db", Stage: "test-stage"}
+ })
+
+ opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
+ _ timestamp.TimeRange, _ mockTSTableOpener, _ any,
+ ) (mockTSTable, error) {
+ return mockTSTable{ID: common.ShardID(0)}, nil
+ },
+ ShardNum: 1,
+ SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
+ TTL: IntervalRule{Unit: DAY, Num: 7},
+ SeriesIndexFlushTimeoutSeconds: 10,
+ SeriesIndexCacheMaxBytes: 1024 * 1024,
+ }
+
+ serviceCache := NewServiceCache().(*serviceCache)
+ sc := newSegmentController[mockTSTable, mockTSTableOpener](
+ ctx, tempDir, l, opts, nil, nil, time.Second,
+ fs.NewLocalFileSystemWithLoggerAndLimit(logger.GetLogger("storage"), opts.MemoryLimit),
+ serviceCache, group,
+ )
+
+ now := time.Now().UTC()
+ startTime := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
+ endTime := startTime.Add(24 * time.Hour)
+
+ segPath := filepath.Join(tempDir, "segment-"+startTime.Format(dayFormat))
+ require.NoError(t, os.MkdirAll(segPath, DirPerm))
+ require.NoError(t, os.WriteFile(
+ filepath.Join(segPath, metadataFilename),
+ []byte(currentVersion), FilePerm))
+
+ seg, err := sc.openSegment(ctx, startTime, endTime, segPath,
+ startTime.Format(dayFormat), sc.groupCache)
+ require.NoError(t, err)
+
+ // Idle-close: drop the only reference so performCleanup runs and the
+ // segment ends up in the residual state seen on cold-tier nodes
+ // (refCount=0, s.index=nil, but segment still reachable for reopen).
+ seg.DecRef()
+ require.Nil(t, seg.index)
+ require.Equal(t, int32(0), atomic.LoadInt32(&seg.refCount))
+
+ indexDB := seg.IndexDB()
+ require.True(t, indexDB == nil,
+ "IndexDB() must return untyped nil after idle close so the caller's "+
+ "`if indexDB == nil` guard fires")
+
+ // Drive the exact caller pattern from collectSeriesIndexInfo. It must
+ // short-circuit on the nil check and never dispatch Stats() on a nil
+ // receiver.
+ var dataCount, dataSize int64
+ func() {
+ defer func() {
+ if r := recover(); r != nil {
+ t.Fatalf("caller pattern must not panic but got: %v", r)
+ }
+ }()
+ if indexDB != nil {
+ dataCount, dataSize = indexDB.Stats()
+ }
+ }()
+ assert.Equal(t, int64(0), dataCount)
+ assert.Equal(t, int64(0), dataSize)
+}
+
+// TestSegment_ConcurrentReopen_RefCountConsistent stress-tests the
+// reference-count contract of segment.incRef under concurrent reopen.
+// Before the fix, segment.initialize() returned without AddInt32 when
+// another goroutine had already reopened the segment, so the second caller
+// "owned" a reference it had never accounted for. The first DecRef would
+// then trigger performCleanup while another goroutine was still using the
+// segment.
+//
+// After the fix, every concurrent incRef must add exactly one reference
+// regardless of which path it takes.
+//
+// Reliability note: a single 64-goroutine round can occasionally serialize
+// well enough that only the first caller enters the slow path
+// (initialize) and the rest take the fast path (CAS+1). That serialized
+// shape would also pass on the buggy code. To make the test deterministic
+// across machines, the experiment is repeated for `rounds` cycles, each
+// starting from the residual state. The bug becomes effectively
+// impossible to miss across all rounds combined.
+func TestSegment_ConcurrentReopen_RefCountConsistent(t *testing.T) {
+ tempDir, cleanup := setupTestEnvironment(t)
+ defer cleanup()
+
+ ctx := context.Background()
+ l := logger.GetLogger("test-concurrent-reopen")
+ ctx = context.WithValue(ctx, logger.ContextKey, l)
+ ctx = common.SetPosition(ctx, func(_ common.Position) common.Position {
+ return common.Position{Database: "test-db", Stage: "test-stage"}
+ })
+
+ opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
+ _ timestamp.TimeRange, _ mockTSTableOpener, _ any,
+ ) (mockTSTable, error) {
+ return mockTSTable{ID: common.ShardID(0)}, nil
+ },
+ ShardNum: 2,
+ SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
+ TTL: IntervalRule{Unit: DAY, Num: 7},
+ SeriesIndexFlushTimeoutSeconds: 10,
+ SeriesIndexCacheMaxBytes: 1024 * 1024,
+ }
+
+ serviceCache := NewServiceCache().(*serviceCache)
+ sc := newSegmentController[mockTSTable, mockTSTableOpener](
+ ctx, tempDir, l, opts, nil, nil, time.Second,
+ fs.NewLocalFileSystemWithLoggerAndLimit(logger.GetLogger("storage"), opts.MemoryLimit),
+ serviceCache, group,
+ )
+
+ now := time.Now().UTC()
+ startTime := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
+ endTime := startTime.Add(24 * time.Hour)
+
+ segPath := filepath.Join(tempDir, "segment-"+startTime.Format(dayFormat))
+ require.NoError(t, os.MkdirAll(segPath, DirPerm))
+ require.NoError(t, os.WriteFile(
+ filepath.Join(segPath, metadataFilename),
+ []byte(currentVersion), FilePerm))
+
+ seg, err := sc.openSegment(ctx, startTime, endTime, segPath,
+ startTime.Format(dayFormat), sc.groupCache)
+ require.NoError(t, err)
+
+ // Bring the segment into the cold-tier residual state.
+ seg.DecRef()
+ require.Nil(t, seg.index)
+ require.Equal(t, int32(0), atomic.LoadInt32(&seg.refCount))
+
+ const (
+ N = 64
+ rounds = 10
+ )
+ for r := 0; r < rounds; r++ {
+ require.Equal(t, int32(0), atomic.LoadInt32(&seg.refCount),
+ "round %d: segment must start at refCount=0", r)
+ require.Nil(t, seg.index, "round %d: segment.index must start nil", r)
+
+ var (
+ wg sync.WaitGroup
+ start = make(chan struct{})
+ errCnt int64
+ )
+ wg.Add(N)
+ for i := 0; i < N; i++ {
+ go func() {
+ defer wg.Done()
+ <-start
+ if incErr := seg.incRef(ctx); incErr != nil {
+ atomic.AddInt64(&errCnt, 1)
+ }
+ }()
+ }
+ close(start)
+ wg.Wait()
+
+ require.Equal(t, int64(0), atomic.LoadInt64(&errCnt),
+ "round %d: incRef must not error", r)
+ require.Equal(t, int32(N), atomic.LoadInt32(&seg.refCount),
+ "round %d: every concurrent incRef must add exactly one reference", r)
+ require.NotNil(t, seg.index, "round %d: segment.index must be reopened", r)
+
+ // Drain references; the last DecRef must trigger performCleanup
+ // and bring the segment back to the residual state for the next
+ // round.
+ for i := 0; i < N; i++ {
+ seg.DecRef()
+ }
+ require.Equal(t, int32(0), atomic.LoadInt32(&seg.refCount),
+ "round %d", r)
+ require.Nil(t, seg.index,
+ "round %d: segment.index must be cleared by performCleanup", r)
+ }
+}
+
+// TestSegment_ConcurrentReopenAndClose_NoPanic exercises the cold-tier
+// reopen pattern under heavy concurrency: many goroutines repeatedly call
+// selectSegments, exercise IndexDB().Stats() on each returned segment, and
+// release with DecRef. With idle-closed segments as the starting state,
+// each iteration triggers initialize() to reopen the segment and the final
+// DecRef triggers performCleanup() to close it again, so closes and
+// reopens are interleaved across goroutines.
+//
+// Verifies (run with -race):
+// - no goroutine panics (covers both the typed-nil and refcount fixes)
+// - no data race is detected on the segment lifecycle
+// - all segments end up cleanly closed (refCount == 0, index == nil)
+func TestSegment_ConcurrentReopenAndClose_NoPanic(t *testing.T) {
+ tempDir, cleanup := setupTestEnvironment(t)
+ defer cleanup()
+
+ ctx := context.Background()
+ l := logger.GetLogger("test-concurrent-reopen-close")
+ ctx = context.WithValue(ctx, logger.ContextKey, l)
+ ctx = common.SetPosition(ctx, func(_ common.Position) common.Position {
+ return common.Position{Database: "test-db", Stage: "test-stage"}
+ })
+
+ opts := TSDBOpts[mockTSTable, mockTSTableOpener]{
+ TSTableCreator: func(_ fs.FileSystem, _ string, _ common.Position, _ *logger.Logger,
+ _ timestamp.TimeRange, _ mockTSTableOpener, _ any,
+ ) (mockTSTable, error) {
+ return mockTSTable{ID: common.ShardID(0)}, nil
+ },
+ ShardNum: 2,
+ SegmentInterval: IntervalRule{Unit: DAY, Num: 1},
+ TTL: IntervalRule{Unit: DAY, Num: 7},
+ SeriesIndexFlushTimeoutSeconds: 10,
+ SeriesIndexCacheMaxBytes: 1024 * 1024,
+ }
+
+ serviceCache := NewServiceCache().(*serviceCache)
+ sc := newSegmentController[mockTSTable, mockTSTableOpener](
+ ctx, tempDir, l, opts, nil, nil, time.Second,
+ fs.NewLocalFileSystemWithLoggerAndLimit(logger.GetLogger("storage"), opts.MemoryLimit),
+ serviceCache, group,
+ )
+
+ now := time.Now().UTC()
+ day1 := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
+ day2 := day1.Add(24 * time.Hour)
+ day3 := day2.Add(24 * time.Hour)
+ days := []time.Time{day1, day2, day3}
+
+ for _, d := range days {
+ segPath := filepath.Join(tempDir, "segment-"+d.Format(dayFormat))
+ require.NoError(t, os.MkdirAll(segPath, DirPerm))
+ require.NoError(t, os.WriteFile(
+ filepath.Join(segPath, metadataFilename),
+ []byte(currentVersion), FilePerm))
+ seg, openErr := sc.openSegment(ctx, d, d.Add(24*time.Hour),
+ segPath, d.Format(dayFormat), sc.groupCache)
+ require.NoError(t, openErr)
+ sc.Lock()
+ sc.lst = append(sc.lst, seg)
+ sc.sortLst()
+ sc.Unlock()
+ // Drop the open reference so each segment starts in the residual state.
+ seg.DecRef()
+ require.Nil(t, seg.index)
+ }
+
+ timeRange := timestamp.NewInclusiveTimeRange(day1, day3.Add(24*time.Hour))
+
+ const (
+ goroutines = 8
+ cycles = 200
+ )
+ var (
+ wg sync.WaitGroup
+ start = make(chan struct{})
+ panicCnt int64
+ )
+ for i := 0; i < goroutines; i++ {
+ wg.Add(1)
+ go func(id int) {
+ defer wg.Done()
+ defer func() {
+ if r := recover(); r != nil {
+ atomic.AddInt64(&panicCnt, 1)
+ t.Errorf("goroutine-%d panicked: %v", id, r)
+ }
+ }()
+ <-start
+ for c := 0; c < cycles; c++ {
+ segs, selErr := sc.selectSegments(timeRange)
+ if selErr != nil {
+ t.Errorf("goroutine-%d cycle %d selectSegments: %v", id, c, selErr)
+ return
+ }
+ for _, s := range segs {
+ if idx := s.IndexDB(); idx != nil {
+ _, _ = idx.Stats()
+ }
+ s.DecRef()
+ }
+ }
+ }(i)
+ }
+ close(start)
+ wg.Wait()
+
+ require.Equal(t, int64(0), atomic.LoadInt64(&panicCnt),
+ "no goroutine should panic during the concurrent reopen/close cycle")
+
+ sc.RLock()
+ final := append([]*segment[mockTSTable, mockTSTableOpener]{}, sc.lst...)
+ sc.RUnlock()
+ for _, s := range final {
+ require.Equal(t, int32(0), atomic.LoadInt32(&s.refCount),
+ "segment %s leaked references", s.suffix)
+ require.Nil(t, s.index, "segment %s should be cleaned up", s.suffix)
+ }
+}
diff --git a/banyand/liaison/grpc/deletion.go b/banyand/liaison/grpc/deletion.go
index 60edfd033..5bc457086 100644
--- a/banyand/liaison/grpc/deletion.go
+++ b/banyand/liaison/grpc/deletion.go
@@ -21,6 +21,7 @@ import (
"context"
"errors"
"fmt"
+ "strings"
"sync"
"time"
@@ -158,11 +159,16 @@ func (m *groupDeletionTaskManager) startDeletion(ctx context.Context, group stri
return fmt.Errorf("deletion task for group %s is already in progress", group)
}
- dataInfo, dataErr := m.schemaRegistry.CollectDataInfo(ctx, group)
+ dataInfo, collectionErrs, dataErr := m.schemaRegistry.CollectDataInfo(ctx, group)
if dataErr != nil {
m.tasks.Delete(group)
return fmt.Errorf("failed to collect data info for group %s: %w", group, dataErr)
}
+ if len(collectionErrs) > 0 {
+ m.tasks.Delete(group)
+ return fmt.Errorf("incomplete data info for group %s, refusing to start deletion: %s",
+ group, strings.Join(collectionErrs, "; "))
+ }
var totalDataSize int64
for _, di := range dataInfo {
totalDataSize += di.GetDataSizeBytes()
@@ -452,10 +458,14 @@ func (m *groupDeletionTaskManager) getDeletionTask(ctx context.Context, group st
}
func (m *groupDeletionTaskManager) hasNonEmptyResources(ctx context.Context, group string) (bool, error) {
- dataInfo, dataErr := m.schemaRegistry.CollectDataInfo(ctx, group)
+ dataInfo, collectionErrs, dataErr := m.schemaRegistry.CollectDataInfo(ctx, group)
if dataErr != nil {
return false, fmt.Errorf("failed to collect data info: %w", dataErr)
}
+ if len(collectionErrs) > 0 {
+ return false, fmt.Errorf("incomplete data info for group %s, cannot determine emptiness: %s",
+ group, strings.Join(collectionErrs, "; "))
+ }
for _, di := range dataInfo {
if di.GetDataSizeBytes() > 0 {
return true, nil
diff --git a/banyand/liaison/grpc/deletion_test.go b/banyand/liaison/grpc/deletion_test.go
index 88b810a7c..955a23abd 100644
--- a/banyand/liaison/grpc/deletion_test.go
+++ b/banyand/liaison/grpc/deletion_test.go
@@ -121,7 +121,7 @@ func TestHasNonEmptyResources(t *testing.T) {
defer ctrl.Finish()
mockRepo := metadata.NewMockRepo(ctrl)
- mockRepo.EXPECT().CollectDataInfo(gomock.Any(), "test-group").Return(tt.infos, nil)
+ mockRepo.EXPECT().CollectDataInfo(gomock.Any(), "test-group").Return(tt.infos, nil, nil)
m := &groupDeletionTaskManager{schemaRegistry: mockRepo}
hasResources, checkErr := m.hasNonEmptyResources(context.Background(), "test-group")
@@ -129,6 +129,30 @@ func TestHasNonEmptyResources(t *testing.T) {
assert.Equal(t, tt.expected, hasResources)
})
}
+
+ // Partial-failure path: when CollectDataInfo reports per-node errors,
+ // hasNonEmptyResources must refuse to answer (returning an error)
+ // rather than silently treating the half-populated DataInfo as
+ // authoritative. A wrong "false" here would let the caller delete
+ // data on the silent node.
+ t.Run("partial failure refuses to answer", func(t *testing.T) {
+ ctrl := gomock.NewController(t)
+ defer ctrl.Finish()
+
+ mockRepo := metadata.NewMockRepo(ctrl)
+ mockRepo.EXPECT().CollectDataInfo(gomock.Any(), "test-group").Return(
+ []*databasev1.DataInfo{{DataSizeBytes: 0}}, // looks empty
+ []string{"node error: cold-0 panic"}, // but cold-0 did not report
+ nil,
+ )
+
+ m := &groupDeletionTaskManager{schemaRegistry: mockRepo}
+ hasResources, checkErr := m.hasNonEmptyResources(context.Background(), "test-group")
+ require.Error(t, checkErr, "partial collection failure must surface as an error, not (false, nil)")
+ assert.Contains(t, checkErr.Error(), "incomplete data info")
+ assert.Contains(t, checkErr.Error(), "cold-0 panic")
+ assert.False(t, hasResources, "the bool return is irrelevant when the error is non-nil; caller must check err first")
+ })
}
func TestDeletion(t *testing.T) {
@@ -192,7 +216,7 @@ func TestDeletion(t *testing.T) {
}
mockRepo := metadata.NewMockRepo(ctrl)
- mockRepo.EXPECT().CollectDataInfo(gomock.Any(), group).Return([]*databasev1.DataInfo{{DataSizeBytes: 512}}, nil)
+ mockRepo.EXPECT().CollectDataInfo(gomock.Any(), group).Return([]*databasev1.DataInfo{{DataSizeBytes: 512}}, nil, nil)
mockRepo.EXPECT().IndexRuleBindingRegistry().Return(&stubIndexRuleBinding{})
mockRepo.EXPECT().IndexRuleRegistry().Return(&stubIndexRule{})
diff --git a/banyand/liaison/grpc/registry.go b/banyand/liaison/grpc/registry.go
index 1e9af0c15..33977c4bc 100644
--- a/banyand/liaison/grpc/registry.go
+++ b/banyand/liaison/grpc/registry.go
@@ -20,6 +20,8 @@ package grpc
import (
"context"
"errors"
+ "fmt"
+ "strings"
"time"
"google.golang.org/grpc/codes"
@@ -754,11 +756,16 @@ func (rs *groupRegistryServer) Inspect(ctx context.Context, req *databasev1.Grou
rs.metrics.totalRegistryErr.Inc(1, g, "group", "inspect")
return nil, schemaErr
}
- dataInfo, dataErr := rs.schemaRegistry.CollectDataInfo(ctx, g)
+ dataInfo, collectionErrs, dataErr := rs.schemaRegistry.CollectDataInfo(ctx, g)
if dataErr != nil {
rs.metrics.totalRegistryErr.Inc(1, g, "group", "inspect")
return nil, dataErr
}
+ if len(collectionErrs) > 0 {
+ rs.metrics.totalRegistryErr.Inc(1, g, "group", "inspect")
+ return nil, fmt.Errorf("incomplete data info for group %s: %s",
+ g, strings.Join(collectionErrs, "; "))
+ }
liaisonInfo, liaisonErr := rs.schemaRegistry.CollectLiaisonInfo(ctx, g)
if liaisonErr != nil {
rs.metrics.totalRegistryErr.Inc(1, g, "group", "inspect")
diff --git a/banyand/measure/metadata_internal_test.go b/banyand/measure/metadata_internal_test.go
new file mode 100644
index 000000000..87fa5db3a
--- /dev/null
+++ b/banyand/measure/metadata_internal_test.go
@@ -0,0 +1,48 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+)
+
+// nilIndexDBSegment is a stub Segment whose IndexDB() returns an untyped
+// nil. The embedded interface lets us satisfy the Segment[*tsTable, option]
+// signature without implementing every method (only IndexDB is exercised
+// by collectSeriesIndexInfo).
+type nilIndexDBSegment struct {
+ storage.Segment[*tsTable, option]
+}
+
+func (nilIndexDBSegment) IndexDB() storage.IndexDB { return nil }
+
+// TestSchemaRepo_CollectSeriesIndexInfo_NilIndexDB locks in the contract
+// that collectSeriesIndexInfo returns an empty SeriesIndexInfo (no panic)
+// when the segment is in the cold-tier residual state where its underlying
+// series index has been torn down by performCleanup.
+func TestSchemaRepo_CollectSeriesIndexInfo_NilIndexDB(t *testing.T) {
+ sr := &schemaRepo{}
+ info := sr.collectSeriesIndexInfo(nilIndexDBSegment{})
+ require.NotNil(t, info)
+ require.Equal(t, int64(0), info.DataCount)
+ require.Equal(t, int64(0), info.DataSizeBytes)
+}
diff --git a/banyand/metadata/client.go b/banyand/metadata/client.go
index 9ae0dd359..88b43bad7 100644
--- a/banyand/metadata/client.go
+++ b/banyand/metadata/client.go
@@ -665,7 +665,7 @@ func (s *clientService) Subjects(ctx context.Context, indexRule *databasev1.Inde
return foundSubjects, subjectErr
}
-func (s *clientService) CollectDataInfo(ctx context.Context, group string) ([]*databasev1.DataInfo, error) {
+func (s *clientService) CollectDataInfo(ctx context.Context, group string) ([]*databasev1.DataInfo, []string, error) {
return s.infoCollectorRegistry.CollectDataInfo(ctx, group)
}
diff --git a/banyand/metadata/metadata.go b/banyand/metadata/metadata.go
index ac5b6508c..35a6644d8 100644
--- a/banyand/metadata/metadata.go
+++ b/banyand/metadata/metadata.go
@@ -53,7 +53,7 @@ type Repo interface {
RegisterHandler(string, schema.Kind, schema.EventHandler)
NodeRegistry() schema.Node
PropertyRegistry() schema.Property
- CollectDataInfo(context.Context, string) ([]*databasev1.DataInfo, error)
+ CollectDataInfo(context.Context, string) ([]*databasev1.DataInfo, []string, error)
CollectLiaisonInfo(context.Context, string) ([]*databasev1.LiaisonInfo, error)
DropGroup(ctx context.Context, catalog commonv1.Catalog, group string) error
}
diff --git a/banyand/metadata/schema/collector.go b/banyand/metadata/schema/collector.go
index 9b297fd66..6ece113de 100644
--- a/banyand/metadata/schema/collector.go
+++ b/banyand/metadata/schema/collector.go
@@ -39,6 +39,20 @@ import (
// merge/flush load can take several seconds) while still bounding the call.
const inspectBroadcastTimeout = 30 * time.Second
+// Prefixes used by every producer that writes into
+// GroupLifecycleInfo.errors so downstream consumers can categorize
+// failures via a stable prefix match. The full vocabulary lives here:
+// BroadcastErrorPrefix / FutureErrorPrefix / NodeErrorPrefix are written
+// by broadcastCollectDataInfo below; TopLevelErrorPrefix is written by
+// the InspectAll RPC handler in banyand/queue/sub/group_lifecycle.go,
+// which observes the top-level error returned by CollectDataInfo.
+const (
+ BroadcastErrorPrefix = "broadcast failed: "
+ FutureErrorPrefix = "future error: "
+ NodeErrorPrefix = "node error: "
+ TopLevelErrorPrefix = "top-level: "
+)
+
// GroupGetter provides method to get group metadata.
type GroupGetter interface {
GetGroup(ctx context.Context, group string) (*commonv1.Group, error)
@@ -72,22 +86,29 @@ func NewInfoCollectorRegistry(l *logger.Logger, groupGetter GroupGetter) *InfoCo
}
}
-// CollectDataInfo collects data information from both local and remote data nodes.
-func (icr *InfoCollectorRegistry) CollectDataInfo(ctx context.Context, group string) ([]*databasev1.DataInfo, error) {
+// CollectDataInfo collects data information from both local and remote data
+// nodes. Returns the aggregated DataInfo slice plus a separate slice of
+// per-node collection errors observed during broadcast (broadcast failure,
+// future errors, listener-side common.Error responses); the two slices
+// are independent -- the errors slice only contains failures and is not
+// positionally aligned with DataInfo. The error return is reserved for
+// top-level failures that prevent any aggregation (GetGroup failure,
+// local collector failure, unsupported catalog).
+func (icr *InfoCollectorRegistry) CollectDataInfo(ctx context.Context, group string) ([]*databasev1.DataInfo, []string, error) {
g, getErr := icr.groupGetter.GetGroup(ctx, group)
if getErr != nil {
- return nil, getErr
+ return nil, nil, getErr
}
localInfo, localErr := icr.collectDataInfoLocal(ctx, g.Catalog, group)
if localErr != nil {
- return nil, localErr
+ return nil, nil, localErr
}
localInfoList := []*databasev1.DataInfo{}
if localInfo != nil {
localInfoList = []*databasev1.DataInfo{localInfo}
}
if icr.dataBroadcaster == nil {
- return localInfoList, nil
+ return localInfoList, nil, nil
}
var topic bus.Topic
@@ -99,25 +120,27 @@ func (icr *InfoCollectorRegistry) CollectDataInfo(ctx context.Context, group str
case commonv1.Catalog_CATALOG_TRACE:
topic = data.TopicTraceCollectDataInfo
default:
- return nil, fmt.Errorf("unsupported catalog type: %v", g.Catalog)
+ return nil, nil, fmt.Errorf("unsupported catalog type: %v", g.Catalog)
}
- remoteInfo := icr.broadcastCollectDataInfo(topic, group)
- return append(localInfoList, remoteInfo...), nil
+ remoteInfo, collectionErrors := icr.broadcastCollectDataInfo(topic, group)
+ return append(localInfoList, remoteInfo...), collectionErrors, nil
}
-func (icr *InfoCollectorRegistry) broadcastCollectDataInfo(topic bus.Topic, group string) []*databasev1.DataInfo {
+func (icr *InfoCollectorRegistry) broadcastCollectDataInfo(topic bus.Topic, group string) ([]*databasev1.DataInfo, []string) {
message := bus.NewMessage(bus.MessageID(time.Now().UnixNano()), &databasev1.GroupRegistryServiceInspectRequest{Group: group})
futures, broadcastErr := icr.dataBroadcaster.Broadcast(inspectBroadcastTimeout, topic, message)
if broadcastErr != nil {
icr.l.Warn().Err(broadcastErr).Str("group", group).Msg("failed to broadcast collect data info request")
- return []*databasev1.DataInfo{}
+ return []*databasev1.DataInfo{}, []string{BroadcastErrorPrefix + broadcastErr.Error()}
}
dataInfoList := make([]*databasev1.DataInfo, 0, len(futures))
+ var collectionErrors []string
for _, future := range futures {
msg, getErr := future.Get()
if getErr != nil {
icr.l.Warn().Err(getErr).Str("group", group).Msg("failed to get collect data info response")
+ collectionErrors = append(collectionErrors, FutureErrorPrefix+getErr.Error())
continue
}
msgData := msg.Data()
@@ -128,9 +151,10 @@ func (icr *InfoCollectorRegistry) broadcastCollectDataInfo(topic bus.Topic, grou
}
case *common.Error:
icr.l.Warn().Str("error", d.Error()).Str("group", group).Msg("error collecting data info from node")
+ collectionErrors = append(collectionErrors, NodeErrorPrefix+d.Error())
}
}
- return dataInfoList
+ return dataInfoList, collectionErrors
}
func (icr *InfoCollectorRegistry) collectDataInfoLocal(ctx context.Context, catalog commonv1.Catalog, group string) (*databasev1.DataInfo, error) {
diff --git a/banyand/queue/sub/group_lifecycle.go b/banyand/queue/sub/group_lifecycle.go
index 8104015da..d8d1f495b 100644
--- a/banyand/queue/sub/group_lifecycle.go
+++ b/banyand/queue/sub/group_lifecycle.go
@@ -24,6 +24,7 @@ import (
commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
+ "github.com/apache/skywalking-banyandb/banyand/metadata/schema"
)
// maxInspectGroupConcurrency caps how many groups InspectAll will inspect
@@ -77,12 +78,19 @@ func (s *server) inspectGroup(ctx context.Context, group *commonv1.Group) *fodcv
Catalog: catalogToString(group.Catalog),
ResourceOpts: group.ResourceOpts,
}
- dataInfo, err := s.metadataRepo.CollectDataInfo(ctx, groupName)
+ // CollectDataInfo's contract: a non-nil err means no DataInfo and nil
+ // collectionErrs (per banyand/metadata/schema/collector.go); a nil err
+ // means DataInfo is populated and collectionErrs may carry per-node
+ // broadcast failures. The two paths never overlap, so the top-level
+ // branch sets a fresh single-element Errors slice.
+ dataInfo, collectionErrs, err := s.metadataRepo.CollectDataInfo(ctx, groupName)
if err != nil {
s.log.Warn().Err(err).Str("group", groupName).Msg("Failed to collect data info")
- } else {
- info.DataInfo = dataInfo
+ info.Errors = []string{schema.TopLevelErrorPrefix + err.Error()}
+ return info
}
+ info.DataInfo = dataInfo
+ info.Errors = collectionErrs
return info
}
diff --git a/banyand/queue/sub/group_lifecycle_cold_tier_test.go b/banyand/queue/sub/group_lifecycle_cold_tier_test.go
new file mode 100644
index 000000000..46beacd4c
--- /dev/null
+++ b/banyand/queue/sub/group_lifecycle_cold_tier_test.go
@@ -0,0 +1,298 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package sub
+
+import (
+ "context"
+ "fmt"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
+ "github.com/apache/skywalking-banyandb/banyand/metadata"
+ "github.com/apache/skywalking-banyandb/banyand/metadata/schema"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+// coldTierRepo is a metadata.Repo stub modeled on cold-tier data nodes.
+// CollectDataInfo simulates the post-fix output of schemaRepo.collectSeriesIndexInfo
+// for idle-closed segments: returns DataInfo whose SegmentInfo entries each carry
+// an empty SeriesIndexInfo (DataCount=0, DataSizeBytes=0). The hook lets tests
+// inject failures, latency, or per-call mutation to mimic the cold-tier
+// "InspectAll while closeIdleSegments is firing" race window.
+type coldTierRepo struct {
+ metadata.Repo
+ groupRegistry *mockGroupRegistry
+ hook func(group string, callIdx int) ([]*databasev1.DataInfo, []string, error)
+ callsByGroup sync.Map // map[string]*atomic.Int64
+ totalCalls atomic.Int64
+ concurrentNow atomic.Int32
+ concurrentMax atomic.Int32
+ panicCount atomic.Int32
+}
+
+func (r *coldTierRepo) GroupRegistry() schema.Group {
+ return r.groupRegistry
+}
+
+func (r *coldTierRepo) CollectDataInfo(_ context.Context, group string) (out []*databasev1.DataInfo, collectionErrs []string, err error) {
+ defer func() {
+ if rec := recover(); rec != nil {
+ r.panicCount.Add(1)
+ err = fmt.Errorf("panic in CollectDataInfo: %v", rec)
+ }
+ }()
+ r.totalCalls.Add(1)
+ current := r.concurrentNow.Add(1)
+ defer r.concurrentNow.Add(-1)
+ for {
+ hi := r.concurrentMax.Load()
+ if current <= hi || r.concurrentMax.CompareAndSwap(hi, current) {
+ break
+ }
+ }
+ cntAny, _ := r.callsByGroup.LoadOrStore(group, new(atomic.Int64))
+ idx := cntAny.(*atomic.Int64).Add(1)
+ if r.hook != nil {
+ return r.hook(group, int(idx-1))
+ }
+ return coldTierEmptyDataInfo(), nil, nil
+}
+
+// coldTierEmptyDataInfo mirrors the DataInfo shape the fixed schemaRepo
+// returns when every selected segment is in the idle-closed residual state
+// (s.index = nil): non-nil top-level DataInfo with one SegmentInfo entry whose
+// SeriesIndexInfo is the zero value.
+func coldTierEmptyDataInfo() []*databasev1.DataInfo {
+ return []*databasev1.DataInfo{
+ {
+ SegmentInfo: []*databasev1.SegmentInfo{
+ {
+ SegmentId: "seg-cold-tier-idle",
+ SeriesIndexInfo: &databasev1.SeriesIndexInfo{},
+ ShardInfo: []*databasev1.ShardInfo{},
+ },
+ },
+ DataSizeBytes: 0,
+ },
+ }
+}
+
+func newColdTierGroups(catalogs ...commonv1.Catalog) []*commonv1.Group {
+ groups := make([]*commonv1.Group, 0, len(catalogs))
+ for i, cat := range catalogs {
+ groups = append(groups, &commonv1.Group{
+ Metadata: &commonv1.Metadata{Name: fmt.Sprintf("cold_group_%d", i)},
+ Catalog: cat,
+ ResourceOpts: &commonv1.ResourceOpts{
+ ShardNum: 2,
+ SegmentInterval: &commonv1.IntervalRule{
+ Unit: commonv1.IntervalRule_UNIT_DAY, Num: 1,
+ },
+ Ttl: &commonv1.IntervalRule{
+ Unit: commonv1.IntervalRule_UNIT_DAY, Num: 30,
+ },
+ },
+ })
+ }
+ return groups
+}
+
+// TestInspectAll_ColdTier_Integration simulates the cold-tier traffic
+// pattern that triggered the production nil-pointer panic on
+// demo-banyandb-data-cold-0. FODC InspectAll fan-out plus
+// closeIdleSegments leave segments in the "refCount=0, index=nil"
+// residual state. With the typed-nil and refcount fixes applied,
+// schemaRepo.collectSeriesIndexInfo short-circuits and returns an empty
+// SeriesIndexInfo rather than panicking.
+//
+// This test wires the real InspectAll RPC handler against a stub
+// metadata.Repo that mirrors the post-fix output of the storage path,
+// then drives four cold-tier scenarios end-to-end:
+//
+// A. single InspectAll after idle-close
+// B. repeated InspectAll cycles (cold-tier reinspect)
+// C. concurrent InspectAll calls
+// D. concurrent InspectAll with simulated idle-close churn between calls
+func TestInspectAll_ColdTier_Integration(t *testing.T) {
+ groups := newColdTierGroups(
+ commonv1.Catalog_CATALOG_MEASURE,
+ commonv1.Catalog_CATALOG_STREAM,
+ commonv1.Catalog_CATALOG_TRACE,
+ )
+
+ t.Run("A_SingleInspectAfterIdleClose", func(t *testing.T) {
+ repo := &coldTierRepo{groupRegistry: &mockGroupRegistry{groups: groups}}
+ s := &server{log: logger.GetLogger("cold-tier-A"), metadataRepo: repo}
+
+ resp, err := s.InspectAll(context.Background(), &fodcv1.InspectAllRequest{})
+ require.NoError(t, err)
+ require.NotNil(t, resp)
+ require.Len(t, resp.Groups, len(groups))
+
+ for _, g := range resp.Groups {
+ require.NotEmpty(t, g.DataInfo, "group %s should expose post-fix DataInfo", g.Name)
+ for _, di := range g.DataInfo {
+ require.NotNil(t, di)
+ for _, seg := range di.SegmentInfo {
+ require.NotNil(t, seg.SeriesIndexInfo,
+ "every segment must carry a non-nil (possibly empty) SeriesIndexInfo")
+ assert.Equal(t, int64(0), seg.SeriesIndexInfo.DataCount)
+ assert.Equal(t, int64(0), seg.SeriesIndexInfo.DataSizeBytes)
+ }
+ }
+ }
+ assert.Equal(t, int32(0), repo.panicCount.Load(), "no goroutine should panic")
+ })
+
+ t.Run("B_RepeatedInspectAll_ColdTierReinspect", func(t *testing.T) {
+ repo := &coldTierRepo{groupRegistry: &mockGroupRegistry{groups: groups}}
+ s := &server{log: logger.GetLogger("cold-tier-B"), metadataRepo: repo}
+
+ const cycles = 25
+ for i := 0; i < cycles; i++ {
+ resp, err := s.InspectAll(context.Background(), &fodcv1.InspectAllRequest{})
+ require.NoError(t, err, "cycle %d", i)
+ require.Len(t, resp.Groups, len(groups), "cycle %d returned wrong group count", i)
+ }
+
+ // Every group must have been visited exactly `cycles` times -- no state
+ // leaks across repeated InspectAll calls (this is the "再次获取" check).
+ for _, g := range groups {
+ cntAny, ok := repo.callsByGroup.Load(g.Metadata.Name)
+ require.True(t, ok, "group %s missing call counter", g.Metadata.Name)
+ assert.Equal(t, int64(cycles), cntAny.(*atomic.Int64).Load(),
+ "group %s expected exactly %d calls", g.Metadata.Name, cycles)
+ }
+ assert.Equal(t, int64(cycles*len(groups)), repo.totalCalls.Load())
+ assert.Equal(t, int32(0), repo.panicCount.Load())
+ })
+
+ t.Run("C_ConcurrentInspectAll", func(t *testing.T) {
+ repo := &coldTierRepo{groupRegistry: &mockGroupRegistry{groups: groups}}
+ s := &server{log: logger.GetLogger("cold-tier-C"), metadataRepo: repo}
+
+ const callers = 8
+ var (
+ wg sync.WaitGroup
+ start = make(chan struct{})
+ panicCnt int64
+ )
+ for i := 0; i < callers; i++ {
+ wg.Add(1)
+ go func(id int) {
+ defer wg.Done()
+ defer func() {
+ if r := recover(); r != nil {
+ atomic.AddInt64(&panicCnt, 1)
+ t.Errorf("caller-%d panicked: %v", id, r)
+ }
+ }()
+ <-start
+ resp, err := s.InspectAll(context.Background(), &fodcv1.InspectAllRequest{})
+ if !assert.NoError(t, err, "caller-%d", id) {
+ return
+ }
+ if !assert.NotNil(t, resp, "caller-%d", id) {
+ return
+ }
+ assert.Len(t, resp.Groups, len(groups), "caller-%d", id)
+ }(i)
+ }
+ close(start)
+ wg.Wait()
+
+ assert.Equal(t, int64(0), panicCnt, "no caller may panic")
+ assert.Equal(t, int32(0), repo.panicCount.Load())
+ assert.Equal(t, int64(callers*len(groups)), repo.totalCalls.Load())
+ // Per-group fan-out cap is 32, so at least two groups should have
+ // overlapped within at least one InspectAll invocation.
+ assert.GreaterOrEqual(t, repo.concurrentMax.Load(), int32(2),
+ "concurrent InspectAll should overlap at least two CollectDataInfo calls")
+ })
+
+ t.Run("D_ConcurrentInspectAll_WithIdleCloseChurn", func(t *testing.T) {
+ // hook simulates the cold-tier race window: every other call returns
+ // a "fresh idle close just happened" payload (still empty), and a few
+ // calls return a benign error to mimic transient disk pressure.
+ brittleHook := func(group string, callIdx int) ([]*databasev1.DataInfo, []string, error) {
+ if callIdx%17 == 0 && callIdx > 0 {
+ return nil, nil, fmt.Errorf("simulated transient close error on %s call#%d", group, callIdx)
+ }
+ // Yield to amplify scheduling jitter, then return empty info.
+ time.Sleep(50 * time.Microsecond)
+ return coldTierEmptyDataInfo(), nil, nil
+ }
+ repo := &coldTierRepo{
+ groupRegistry: &mockGroupRegistry{groups: groups},
+ hook: brittleHook,
+ }
+ s := &server{log: logger.GetLogger("cold-tier-D"), metadataRepo: repo}
+
+ const callers = 6
+ const cycles = 20
+ var (
+ wg sync.WaitGroup
+ start = make(chan struct{})
+ panicCnt int64
+ )
+ for i := 0; i < callers; i++ {
+ wg.Add(1)
+ go func(id int) {
+ defer wg.Done()
+ defer func() {
+ if r := recover(); r != nil {
+ atomic.AddInt64(&panicCnt, 1)
+ t.Errorf("caller-%d panicked: %v", id, r)
+ }
+ }()
+ <-start
+ for c := 0; c < cycles; c++ {
+ resp, err := s.InspectAll(context.Background(), &fodcv1.InspectAllRequest{})
+ // require.* would FailNow inside this goroutine, which
+ // is forbidden by the testing package; use assert and
+ // bail out early on the first failure.
+ if !assert.NoError(t, err, "caller-%d cycle-%d", id, c) {
+ return
+ }
+ if !assert.NotNil(t, resp, "caller-%d cycle-%d", id, c) {
+ return
+ }
+ if !assert.Len(t, resp.Groups, len(groups), "caller-%d cycle-%d", id, c) {
+ return
+ }
+ }
+ }(i)
+ }
+ close(start)
+ wg.Wait()
+
+ require.Equal(t, int64(0), panicCnt, "no caller may panic")
+ require.Equal(t, int32(0), repo.panicCount.Load(),
+ "CollectDataInfo must not panic under the cold-tier churn")
+ assert.GreaterOrEqual(t, repo.concurrentMax.Load(), int32(2),
+ "concurrent InspectAll under churn should overlap at least two CollectDataInfo calls")
+ })
+}
diff --git a/banyand/queue/sub/group_lifecycle_test.go b/banyand/queue/sub/group_lifecycle_test.go
index f3a5cd6ff..4f912b323 100644
--- a/banyand/queue/sub/group_lifecycle_test.go
+++ b/banyand/queue/sub/group_lifecycle_test.go
@@ -47,20 +47,21 @@ func (m *mockGroupRegistry) ListGroup(_ context.Context) ([]*commonv1.Group, err
type mockMetadataRepo struct {
metadata.Repo
- groupRegistry *mockGroupRegistry
- dataInfoMap map[string][]*databasev1.DataInfo
- dataInfoErr map[string]error
- slowGroups map[string]time.Duration
- collectStarts atomic.Int32
- concurrentMax atomic.Int32
- concurrentNow atomic.Int32
+ groupRegistry *mockGroupRegistry
+ dataInfoMap map[string][]*databasev1.DataInfo
+ dataInfoErr map[string]error
+ collectionErrors map[string][]string
+ slowGroups map[string]time.Duration
+ collectStarts atomic.Int32
+ concurrentMax atomic.Int32
+ concurrentNow atomic.Int32
}
func (m *mockMetadataRepo) GroupRegistry() schema.Group {
return m.groupRegistry
}
-func (m *mockMetadataRepo) CollectDataInfo(_ context.Context, group string) ([]*databasev1.DataInfo, error) {
+func (m *mockMetadataRepo) CollectDataInfo(_ context.Context, group string) ([]*databasev1.DataInfo, []string, error) {
m.collectStarts.Add(1)
current := m.concurrentNow.Add(1)
defer m.concurrentNow.Add(-1)
@@ -77,13 +78,18 @@ func (m *mockMetadataRepo) CollectDataInfo(_ context.Context, group string) ([]*
}
if m.dataInfoErr != nil {
if err, ok := m.dataInfoErr[group]; ok {
- return nil, err
+ return nil, nil, err
}
}
+ var dataInfo []*databasev1.DataInfo
if m.dataInfoMap != nil {
- return m.dataInfoMap[group], nil
+ dataInfo = m.dataInfoMap[group]
}
- return nil, nil
+ var collectionErrs []string
+ if m.collectionErrors != nil {
+ collectionErrs = m.collectionErrors[group]
+ }
+ return dataInfo, collectionErrs, nil
}
func TestCatalogToString(t *testing.T) {
@@ -264,4 +270,92 @@ func TestInspectAll_RunsGroupsInParallel(t *testing.T) {
assert.GreaterOrEqual(t, repo.concurrentMax.Load(), int32(2),
"at least two CollectDataInfo invocations must overlap; observed max concurrency was %d",
repo.concurrentMax.Load())
+ for _, g := range resp.Groups {
+ assert.Empty(t, g.Errors, "successful group %s must not carry errors", g.Name)
+ }
+}
+
+// TestInspectAll_SurfacesTopLevelError asserts that a top-level
+// CollectDataInfo error is reported through GroupLifecycleInfo.errors with
+// a "top-level:" prefix and that DataInfo stays empty for that group.
+func TestInspectAll_SurfacesTopLevelError(t *testing.T) {
+ repo := &mockMetadataRepo{
+ groupRegistry: &mockGroupRegistry{
+ groups: []*commonv1.Group{
+ {Metadata: &commonv1.Metadata{Name: "ok_group"}, Catalog: commonv1.Catalog_CATALOG_MEASURE},
+ {Metadata: &commonv1.Metadata{Name: "bad_group"}, Catalog: commonv1.Catalog_CATALOG_STREAM},
+ },
+ },
+ dataInfoMap: map[string][]*databasev1.DataInfo{
+ "ok_group": {{DataSizeBytes: 1024}},
+ },
+ dataInfoErr: map[string]error{
+ "bad_group": fmt.Errorf("metadataRepo: simulated load failure"),
+ },
+ }
+ s := &server{log: logger.GetLogger("test"), metadataRepo: repo}
+
+ resp, err := s.InspectAll(context.Background(), &fodcv1.InspectAllRequest{})
+ require.NoError(t, err)
+ require.Len(t, resp.Groups, 2)
+
+ byName := map[string]*fodcv1.GroupLifecycleInfo{}
+ for _, g := range resp.Groups {
+ byName[g.Name] = g
+ }
+
+ ok := byName["ok_group"]
+ require.NotNil(t, ok)
+ assert.Empty(t, ok.Errors, "ok_group must not carry errors")
+ assert.NotEmpty(t, ok.DataInfo)
+
+ bad := byName["bad_group"]
+ require.NotNil(t, bad)
+ assert.Empty(t, bad.DataInfo, "failed group must not expose stale DataInfo")
+ require.NotEmpty(t, bad.Errors, "failed group must surface the CollectDataInfo error")
+ assert.Contains(t, bad.Errors[0], "top-level:",
+ "top-level failures must be tagged with the top-level prefix")
+ assert.Contains(t, bad.Errors[0], "simulated load failure",
+ "the original error message must be preserved")
+}
+
+// TestInspectAll_PropagatesPartialFailureErrors asserts that per-node
+// failures reported by mockMetadataRepo.collectionErrors are passed
+// through to GroupLifecycleInfo.errors for the originating group, while
+// DataInfo still carries the entries from nodes that succeeded.
+func TestInspectAll_PropagatesPartialFailureErrors(t *testing.T) {
+ repo := &mockMetadataRepo{
+ groupRegistry: &mockGroupRegistry{
+ groups: []*commonv1.Group{
+ {Metadata: &commonv1.Metadata{Name: "partial_group"}, Catalog: commonv1.Catalog_CATALOG_MEASURE},
+ },
+ },
+ dataInfoMap: map[string][]*databasev1.DataInfo{
+ "partial_group": {{DataSizeBytes: 512}},
+ },
+ collectionErrors: map[string][]string{
+ "partial_group": {
+ "future error: rpc error: nil pointer dereference",
+ "node error: cold-0 panic",
+ },
+ },
+ }
+ s := &server{log: logger.GetLogger("test"), metadataRepo: repo}
+
+ resp, err := s.InspectAll(context.Background(), &fodcv1.InspectAllRequest{})
+ require.NoError(t, err)
+ require.Len(t, resp.Groups, 1)
+ g := resp.Groups[0]
+ assert.NotEmpty(t, g.DataInfo, "partial success must still expose the DataInfo entries it did get")
+ assert.Equal(t,
+ []string{
+ "future error: rpc error: nil pointer dereference",
+ "node error: cold-0 panic",
+ },
+ g.Errors,
+ "per-node collection errors must be passed through to GroupLifecycleInfo.errors")
+ for _, e := range g.Errors {
+ assert.NotContains(t, e, "top-level:",
+ "partial-success errors must not carry the top-level: prefix")
+ }
}
diff --git a/banyand/stream/metadata_internal_test.go b/banyand/stream/metadata_internal_test.go
new file mode 100644
index 000000000..a292b6704
--- /dev/null
+++ b/banyand/stream/metadata_internal_test.go
@@ -0,0 +1,48 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package stream
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+)
+
+// nilIndexDBSegment is a stub Segment whose IndexDB() returns an untyped
+// nil. The embedded interface lets us satisfy the Segment[*tsTable, option]
+// signature without implementing every method (only IndexDB is exercised
+// by collectSeriesIndexInfo).
+type nilIndexDBSegment struct {
+ storage.Segment[*tsTable, option]
+}
+
+func (nilIndexDBSegment) IndexDB() storage.IndexDB { return nil }
+
+// TestSchemaRepo_CollectSeriesIndexInfo_NilIndexDB locks in the contract
+// that collectSeriesIndexInfo returns an empty SeriesIndexInfo (no panic)
+// when the segment is in the cold-tier residual state where its underlying
+// series index has been torn down by performCleanup.
+func TestSchemaRepo_CollectSeriesIndexInfo_NilIndexDB(t *testing.T) {
+ sr := &schemaRepo{}
+ info := sr.collectSeriesIndexInfo(nilIndexDBSegment{})
+ require.NotNil(t, info)
+ require.Equal(t, int64(0), info.DataCount)
+ require.Equal(t, int64(0), info.DataSizeBytes)
+}
diff --git a/banyand/trace/metadata_internal_test.go b/banyand/trace/metadata_internal_test.go
new file mode 100644
index 000000000..378078907
--- /dev/null
+++ b/banyand/trace/metadata_internal_test.go
@@ -0,0 +1,48 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package trace
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/skywalking-banyandb/banyand/internal/storage"
+)
+
+// nilIndexDBSegment is a stub Segment whose IndexDB() returns an untyped
+// nil. The embedded interface lets us satisfy the Segment[*tsTable, option]
+// signature without implementing every method (only IndexDB is exercised
+// by collectSeriesIndexInfo).
+type nilIndexDBSegment struct {
+ storage.Segment[*tsTable, option]
+}
+
+func (nilIndexDBSegment) IndexDB() storage.IndexDB { return nil }
+
+// TestSchemaRepo_CollectSeriesIndexInfo_NilIndexDB locks in the contract
+// that collectSeriesIndexInfo returns an empty SeriesIndexInfo (no panic)
+// when the segment is in the cold-tier residual state where its underlying
+// series index has been torn down by performCleanup.
+func TestSchemaRepo_CollectSeriesIndexInfo_NilIndexDB(t *testing.T) {
+ sr := &schemaRepo{}
+ info := sr.collectSeriesIndexInfo(nilIndexDBSegment{})
+ require.NotNil(t, info)
+ require.Equal(t, int64(0), info.DataCount)
+ require.Equal(t, int64(0), info.DataSizeBytes)
+}
diff --git a/docs/api-reference.md b/docs/api-reference.md
index 4a5bc34e9..2c12c584f 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -4773,6 +4773,7 @@ Phase represents the current phase of the deletion task.
| catalog | [string](#string) | | |
| resource_opts | [banyandb.common.v1.ResourceOpts](#banyandb-common-v1-ResourceOpts) | | |
| data_info | [banyandb.database.v1.DataInfo](#banyandb-database-v1-DataInfo) | repeated | |
+| errors | [string](#string) | repeated | errors lists every failure observed while collecting data_info for this group: top-level CollectDataInfo failures (GetGroup, missing collector, dial failure -- prefixed "top-level: ") and per-node broadcast failures (prefixed "future error: ", "node error: ", "broadcast failed: "). Combined with len(data_info), consumers can tell the following four states apart: - data_info empty && errors empty -> no no [...]
diff --git a/fodc/proxy/internal/lifecycle/manager.go b/fodc/proxy/internal/lifecycle/manager.go
index 23c7bc9db..d17128b8c 100644
--- a/fodc/proxy/internal/lifecycle/manager.go
+++ b/fodc/proxy/internal/lifecycle/manager.go
@@ -20,6 +20,7 @@ package lifecycle
import (
"context"
+ "sort"
"sync"
fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1"
@@ -258,16 +259,41 @@ func (m *Manager) buildInspectionResult(allData []*agentLifecycleData) *Inspecti
func (m *Manager) mergeGroups(allData []*agentLifecycleData) []*fodcv1.GroupLifecycleInfo {
groupMap := make(map[string]*fodcv1.GroupLifecycleInfo)
+ // Errors are unioned across agents because each agent may observe a
+ // different subset of per-node failures (e.g. liaison-0 sees cold-0
+ // time out while liaison-1 sees cold-0 panic). Last-wins on the rest
+ // of the GroupLifecycleInfo is fine -- name/catalog/resource_opts are
+ // agent-invariant -- but errors must be the deduped union.
+ mergedErrors := make(map[string]map[string]struct{})
for _, ad := range allData {
if ad == nil || ad.Data == nil {
continue
}
for _, g := range ad.Data.Groups {
groupMap[g.Name] = g
+ if len(g.Errors) == 0 {
+ continue
+ }
+ set, ok := mergedErrors[g.Name]
+ if !ok {
+ set = make(map[string]struct{})
+ mergedErrors[g.Name] = set
+ }
+ for _, e := range g.Errors {
+ set[e] = struct{}{}
+ }
}
}
groups := make([]*fodcv1.GroupLifecycleInfo, 0, len(groupMap))
- for _, g := range groupMap {
+ for name, g := range groupMap {
+ if set := mergedErrors[name]; len(set) > 0 {
+ errs := make([]string, 0, len(set))
+ for e := range set {
+ errs = append(errs, e)
+ }
+ sort.Strings(errs)
+ g.Errors = errs
+ }
groups = append(groups, g)
}
return groups
diff --git a/fodc/proxy/internal/lifecycle/manager_test.go b/fodc/proxy/internal/lifecycle/manager_test.go
index 55a76f166..3f1117705 100644
--- a/fodc/proxy/internal/lifecycle/manager_test.go
+++ b/fodc/proxy/internal/lifecycle/manager_test.go
@@ -394,3 +394,64 @@ func TestManager_CollectLifecycle_ContextCanceled(t *testing.T) {
result, _ := mgr.CollectLifecycle(ctx)
require.NotNil(t, result)
}
+
+// TestManager_MergeGroups_UnionsErrorsAcrossAgents verifies that when
+// multiple agents report the same group, the errors observed by each
+// agent are unioned (deduped, sorted) rather than overwritten last-wins,
+// so downstream consumers see every per-node failure regardless of which
+// agent's view they happened to land on.
+func TestManager_MergeGroups_UnionsErrorsAcrossAgents(t *testing.T) {
+ log := initTestLogger(t)
+ mgr := NewManager(nil, nil, log)
+
+ groupName := "sw_metric"
+ agentA := &agentLifecycleData{
+ PodName: "liaison-0",
+ Data: &fodcv1.LifecycleData{
+ Groups: []*fodcv1.GroupLifecycleInfo{{
+ Name: groupName,
+ Catalog: "CATALOG_MEASURE",
+ Errors: []string{
+ "node error: cold-0 panic",
+ "future error: rpc deadline",
+ },
+ }},
+ },
+ }
+ agentB := &agentLifecycleData{
+ PodName: "liaison-1",
+ Data: &fodcv1.LifecycleData{
+ Groups: []*fodcv1.GroupLifecycleInfo{{
+ Name: groupName,
+ Catalog: "CATALOG_MEASURE",
+ Errors: []string{
+ "node error: cold-0 panic", // duplicate of A's first
+ "broadcast failed: dial timeout",
+ },
+ }},
+ },
+ }
+ agentC := &agentLifecycleData{
+ PodName: "liaison-2",
+ Data: &fodcv1.LifecycleData{
+ Groups: []*fodcv1.GroupLifecycleInfo{{
+ Name: groupName,
+ Catalog: "CATALOG_MEASURE",
+ // no errors
+ }},
+ },
+ }
+
+ merged := mgr.mergeGroups([]*agentLifecycleData{agentA, agentB, agentC})
+ require.Len(t, merged, 1)
+ g := merged[0]
+ assert.Equal(t, groupName, g.Name)
+ assert.Equal(t,
+ []string{
+ "broadcast failed: dial timeout",
+ "future error: rpc deadline",
+ "node error: cold-0 panic",
+ },
+ g.Errors,
+ "errors must be the deduped, sorted union of every agent's view")
+}
(skywalking-banyandb) 02/14: fix(test): merge query_ondisk into query and fix flaky on-disk integration tests (#1041)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 18477053b7aa445e2432ea347998e584305d0b0f
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Wed Apr 1 20:07:18 2026 +0800
fix(test): merge query_ondisk into query and fix flaky on-disk integration tests (#1041)
* Merge query_ondisk into query with two-round testing pattern
- Extract test entries and RegisterTable from all case files
- Add Restart function to common.go SetupResult structs
- Convert suites to use ClosableStandalone/EmptyClosableStandalone pattern
- Create round2.go for on-disk testing with fresh gRPC connections
- Delete entire query_ondisk directory as it's now merged
Co-Authored-By: Claude Opus 4.6 <no...@anthropic.com>
---
CHANGES.md | 65 ++++++++++---
test/cases/measure/measure.go | 11 ++-
test/cases/property/property.go | 11 ++-
test/cases/stream/stream.go | 19 ++--
test/cases/topn/topn.go | 5 -
test/cases/trace/trace.go | 19 ++--
.../standalone/multi_segments/common.go | 4 +
.../standalone/multi_segments/etcd/suite_test.go | 24 ++++-
.../multi_segments/property/suite_test.go | 21 ++++-
.../standalone/multi_segments/round2.go | 61 ++++++++++++
test/integration/standalone/query/common.go | 4 +
.../standalone/query/etcd/suite_test.go | 28 +++++-
.../standalone/query/property/suite_test.go | 21 ++++-
test/integration/standalone/query/round2.go | 64 +++++++++++++
test/integration/standalone/query_ondisk/common.go | 103 ---------------------
.../standalone/query_ondisk/etcd/suite_test.go | 64 -------------
.../standalone/query_ondisk/property/suite_test.go | 69 --------------
test_table | Bin 0 -> 2254768 bytes
18 files changed, 313 insertions(+), 280 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index ace8b3034..ecbffb7ee 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -2,25 +2,64 @@
Release Notes.
-<<<<<<< HEAD
## 0.10.2
-=======
-## 0.11.0
+
+### Bug Fixes
+
+- Fix reuse of byte arrays in min/max implementation causing data corruption.
+- Fix index-mode measure queries returning documents outside requested time range.
+- Fix flaky trace query filtering caused by non-deterministic sidx tag ordering and add consistency checks for integration query cases.
+- MCP: Add validation for properties and harden the mcp server.
+- Fix property schema client connection not stable after data node restarted.
+- Fix flaky on-disk integration tests caused by Ginkgo v2 random container shuffling closing gRPC connections prematurely.
+
+## 0.10.0
### Features
-- Organize access logs under a dedicated "accesslog" subdirectory to improve log organization and separation from other application data.
-- Collect BanyanDB data on e2e test failure for CI debugging.
-- Add log query e2e test.
-- Sync lifecycle e2e test from SkyWalking stages test.
-- Add periodic health check for property schema connection.
-- Persist segment end time in per-segment metadata so boundaries don't shift across restarts or config changes.
->>>>>>> 29e87c9d (Stable Segment Endtime (#1051))
+- Remove Bloom filter for dictionary-encoded tags.
+- Implement BanyanDB MCP.
+- Support deleting non-entity tags when updating the schema.
+- Remove check requiring tags in criteria to be present in projection.
+- Add sorted query support for the Property.
+- Update bydbQL to add sorted query support for the Property.
+- Remove the windows arch for binary and docker image.
+- Support writing data with specifications.
+- Persist series metadata in liaison queue for measure, stream and trace models.
+- Update the dump tool to support analyzing the parts with smeta files.
+- Add replication integration test for measure.
+- Activate the property repair mechanism by default.
+- Add snapshot time retention policy to ensure the snapshot only can be deleted after the configured minimum age(time).
+- **Breaking Change**: Change the data storage path structure for property model:
+ - From: `<data-dir>/property/data/shard-<id>/...`
+ - To: `<data-dir>/property/data/<group>/shard-<id>/...`
+- Add a generic snapshot coordination package for atomic snapshot transitions across trace and sidx.
+- Support map-reduce aggregation for measure queries: map phase (partial aggregation on data nodes) and reduce phase (final aggregation on liaison).
+- Add eBPF-based KTM I/O monitor for FODC agent.
+- Support relative paths in configuration.
+- Support 'none' node discovery and make it the default.
+- Support server-side element ID generation for stream writes when clients omit element_id.
+- Implement entire group deletion.
### Bug Fixes
-- Fix reuse of byte arrays in min/max implementation causing data corruption.
-- Fix index-mode measure queries returning documents outside requested time range.
+- Fix the wrong retention setting of each measure/stream/trace.
+- Fix server got panic when create/update property with high dist usage.
+- Fix incorrect key range update in sidx part metadata.
+- Fix panic in measure block merger when merging blocks with overlapping timestamps.
+- Fix unsupported empty string tag bug.
+- Fix duplicate elements in stream query results by implementing element ID-based deduplication across scan, merge, and result building stages.
+- Fix data written to the wrong shard and related stream queries.
+- Fix the lifecycle panic when the trace has no sidx.
+- Fix panic in sidx merge and flush operations when part counts don't match expectations.
+- Fix trace queries with range conditions on the same tag (e.g., duration) combined with ORDER BY by deduplicating tag names when merging logical expression branches.
+- Fix sidx tag filter range check returning inverted skip decision and use correct int64 encoding for block min/max.
+- Ignore take snapshot when no data.
+- Fix measure standalone write handler resetting accumulated groups on error, which dropped all successfully processed events in the batch.
+- Fix memory part reference leak in mustAddMemPart when tsTable loop closes.
+- Fix memory part leak in syncPartContext Close and prevent double-release in FinishSync.
+- Fix segment reference leaks in measure/stream/trace queries and ensure chunked sync sessions close part contexts correctly.
+- Fix duplicate query execution in distributed measure Agg+TopN queries by enabling push-down aggregation, removing the wasteful double-query pattern.
- Fix nil pointer panic in segment collectMetrics during shutdown.
- Fix property schema client connection instability after data node restart.
- Fix take snapshot error when no data in the segment.
@@ -46,4 +85,4 @@ Release Notes.
### Chores
- Upgrade Go and npm dependencies for CVE fixes.
-- Bump ui and mcp npm dependencies for CVE fixes.
\ No newline at end of file
+- Bump ui and mcp npm dependencies for CVE fixes.
diff --git a/test/cases/measure/measure.go b/test/cases/measure/measure.go
index fadf4ee02..743df02b5 100644
--- a/test/cases/measure/measure.go
+++ b/test/cases/measure/measure.go
@@ -39,7 +39,7 @@ var (
}
)
-var _ = g.DescribeTable("Scanning Measures", verify,
+var measureEntries = []any{
g.Entry("filter hidden tag projection", helpers.Args{Input: "filter_hidden_tag", Duration: 25 * time.Minute, Offset: -20 * time.Minute, DisOrder: true}),
g.Entry("index mode filter hidden tag projection", helpers.Args{Input: "index_mode_filter_hidden_tag", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("all", helpers.Args{Input: "all", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
@@ -93,4 +93,11 @@ var _ = g.DescribeTable("Scanning Measures", verify,
g.Entry("filter by non-existent tag", helpers.Args{Input: "filter_non_existent_tag", Duration: 25 * time.Minute, Offset: -20 * time.Minute, WantErr: true}),
g.Entry("project non-existent tag", helpers.Args{Input: "project_non_existent_tag", Duration: 25 * time.Minute, Offset: -20 * time.Minute, WantErr: true}),
g.Entry("write mixed", helpers.Args{Input: "write_mixed", Duration: 15 * time.Minute, Offset: 25 * time.Minute, DisOrder: true}),
-)
+}
+
+// RegisterTable registers the measure test table with the given description.
+func RegisterTable(description string) bool {
+ return g.DescribeTable(description, append([]any{verify}, measureEntries...)...)
+}
+
+var _ = RegisterTable("Scanning Measures")
diff --git a/test/cases/property/property.go b/test/cases/property/property.go
index bb1311553..9b8d2efb5 100644
--- a/test/cases/property/property.go
+++ b/test/cases/property/property.go
@@ -37,7 +37,7 @@ var (
}
)
-var _ = g.DescribeTable("Scanning Properties", verify,
+var propertyEntries = []any{
g.Entry("all", helpers.Args{Input: "all"}),
g.Entry("limit", helpers.Args{Input: "limit"}),
g.Entry("query by criteria", helpers.Args{Input: "query_by_criteria"}),
@@ -47,4 +47,11 @@ var _ = g.DescribeTable("Scanning Properties", verify,
g.Entry("order by with limit", helpers.Args{Input: "order_by_with_limit"}),
g.Entry("order without projection", helpers.Args{Input: "order_without_projection"}),
g.Entry("query with order", helpers.Args{Input: "query_with_order"}),
-)
+}
+
+// RegisterTable registers the property test table with the given description.
+func RegisterTable(description string) bool {
+ return g.DescribeTable(description, append([]any{verify}, propertyEntries...)...)
+}
+
+var _ = RegisterTable("Scanning Properties")
diff --git a/test/cases/stream/stream.go b/test/cases/stream/stream.go
index 662d66285..f9c73e93b 100644
--- a/test/cases/stream/stream.go
+++ b/test/cases/stream/stream.go
@@ -39,11 +39,7 @@ var (
}
)
-var _ = g.DescribeTable("Scanning Streams", func(args helpers.Args) {
- gm.Eventually(func(innerGm gm.Gomega) {
- verify(innerGm, args)
- }, flags.EventuallyTimeout).Should(gm.Succeed())
-},
+var streamEntries = []any{
g.Entry("all elements", helpers.Args{Input: "all", Duration: 1 * time.Hour}),
g.Entry("projection with http.method", helpers.Args{Input: "all_with_http_method", Duration: 1 * time.Hour}),
g.Entry("limit", helpers.Args{Input: "limit", Duration: 1 * time.Hour}),
@@ -99,4 +95,15 @@ var _ = g.DescribeTable("Scanning Streams", func(args helpers.Args) {
g.Entry("filter by non-existent tag", helpers.Args{Input: "filter_non_existent_tag", Duration: 1 * time.Hour, WantErr: true}),
g.Entry("project non-existent tag", helpers.Args{Input: "project_non_existent_tag", Duration: 1 * time.Hour, WantErr: true}),
g.Entry("write mixed", helpers.Args{Input: "write_mixed", Duration: 1 * time.Hour, IgnoreElementID: true}),
-)
+}
+
+// RegisterTable registers the stream test table with the given description.
+func RegisterTable(description string) bool {
+ return g.DescribeTable(description, append([]any{func(args helpers.Args) {
+ gm.Eventually(func(innerGm gm.Gomega) {
+ verify(innerGm, args)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
+ }}, streamEntries...)...)
+}
+
+var _ = RegisterTable("Scanning Streams")
diff --git a/test/cases/topn/topn.go b/test/cases/topn/topn.go
index d08317f13..c1c96eef2 100644
--- a/test/cases/topn/topn.go
+++ b/test/cases/topn/topn.go
@@ -52,11 +52,6 @@ var topnEntries = []any{
g.Entry("using in operation in aggregation", helpers.Args{Input: "in", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("using not-in operation in aggregation", helpers.Args{Input: "not_in", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
g.Entry("max top3 with version merged order by desc", helpers.Args{Input: "aggr_version_merged", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
- g.Entry("int64 max aggregation test", helpers.Args{Input: "topn_max", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
- g.Entry("int64 min aggregation test", helpers.Args{Input: "topn_min", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
- g.Entry("int64 sum aggregation test", helpers.Args{Input: "topn_sum", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
- g.Entry("int64 mean aggregation test", helpers.Args{Input: "topn_mean", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
- g.Entry("int64 count aggregation test", helpers.Args{Input: "topn_count", Duration: 25 * time.Minute, Offset: -20 * time.Minute}),
}
// RegisterTable registers the topn test table with the given description.
diff --git a/test/cases/trace/trace.go b/test/cases/trace/trace.go
index 88b9744d7..474b4f677 100644
--- a/test/cases/trace/trace.go
+++ b/test/cases/trace/trace.go
@@ -37,11 +37,7 @@ var (
}
)
-var _ = g.DescribeTable("Scanning Traces", func(args helpers.Args) {
- gm.Eventually(func(innerGm gm.Gomega) {
- verify(innerGm, args)
- }, flags.EventuallyTimeout).Should(gm.Succeed())
-},
+var traceEntries = []any{
g.Entry("query by trace id", helpers.Args{Input: "eq_trace_id", Duration: 1 * time.Hour}),
g.Entry("query by trace ids", helpers.Args{Input: "in_trace_ids", Duration: 1 * time.Hour}),
g.Entry("query by empty span ids", helpers.Args{Input: "in_empty_span_ids", Duration: 1 * time.Hour, WantEmpty: true}),
@@ -69,4 +65,15 @@ var _ = g.DescribeTable("Scanning Traces", func(args helpers.Args) {
g.Entry("filter by non-existent tag", helpers.Args{Input: "filter_non_existent_tag", Duration: 1 * time.Hour, WantErr: true}),
g.Entry("project non-existent tag", helpers.Args{Input: "project_non_existent_tag", Duration: 1 * time.Hour, WantErr: true}),
g.Entry("write mixed", helpers.Args{Input: "write_mixed", Duration: 1 * time.Hour, DisOrder: true}),
-)
+}
+
+// RegisterTable registers the trace test table with the given description.
+func RegisterTable(description string) bool {
+ return g.DescribeTable(description, append([]any{func(args helpers.Args) {
+ gm.Eventually(func(innerGm gm.Gomega) {
+ verify(innerGm, args)
+ }, flags.EventuallyTimeout).Should(gm.Succeed())
+ }}, traceEntries...)...)
+}
+
+var _ = RegisterTable("Scanning Traces")
diff --git a/test/integration/standalone/multi_segments/common.go b/test/integration/standalone/multi_segments/common.go
index 87e0da611..be60ff34b 100644
--- a/test/integration/standalone/multi_segments/common.go
+++ b/test/integration/standalone/multi_segments/common.go
@@ -41,6 +41,7 @@ import (
// SetupResult contains all info returned by SetupFunc.
type SetupResult struct {
+ Restart func() (string, func())
Now time.Time
BaseTime time.Time
StopFunc func()
@@ -91,6 +92,9 @@ var _ = ginkgo.SynchronizedAfterSuite(func() {
if connection != nil {
gomega.Expect(connection.Close()).To(gomega.Succeed())
}
+ if round2Conn != nil {
+ gomega.Expect(round2Conn.Close()).To(gomega.Succeed())
+ }
}, func() {})
var _ = ginkgo.ReportAfterSuite("Integration Query Suite", func(report ginkgo.Report) {
diff --git a/test/integration/standalone/multi_segments/etcd/suite_test.go b/test/integration/standalone/multi_segments/etcd/suite_test.go
index 9209d5949..f0b7213bc 100644
--- a/test/integration/standalone/multi_segments/etcd/suite_test.go
+++ b/test/integration/standalone/multi_segments/etcd/suite_test.go
@@ -24,6 +24,7 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
+ "github.com/apache/skywalking-banyandb/pkg/test"
"github.com/apache/skywalking-banyandb/pkg/test/setup"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
test_cases "github.com/apache/skywalking-banyandb/test/cases"
@@ -33,16 +34,35 @@ import (
func init() {
multisegments.SetupFunc = func() multisegments.SetupResult {
- addr, _, cleanup := setup.Standalone(nil)
+ path, diskCleanupFn, pathErr := test.NewSpace()
+ Expect(pathErr).NotTo(HaveOccurred())
+ var ports []int
+ ports, portsErr := test.AllocateFreePorts(4)
+ Expect(portsErr).NotTo(HaveOccurred())
+ addr, _, closeFunc := setup.ClosableStandalone(nil, path, ports)
ns := timestamp.NowMilli().UnixNano()
now := time.Unix(0, ns-ns%int64(time.Minute))
baseTime := time.Date(now.Year(), now.Month(), now.Day(), 0o0, 0o2, 0, 0, now.Location())
test_cases.Initialize(addr, baseTime)
+ prevClose, currClose := closeFunc, func() {}
return multisegments.SetupResult{
Addr: addr,
Now: now,
BaseTime: baseTime,
- StopFunc: cleanup,
+ Restart: func() (string, func()) {
+ time.Sleep(5 * time.Second)
+ prevClose()
+ time.Sleep(3 * time.Second)
+ currClose()
+ addr, _, closeFunc := setup.EmptyClosableStandalone(nil, path, ports)
+ prevClose, currClose = currClose, closeFunc
+ return addr, closeFunc
+ },
+ StopFunc: func() {
+ currClose()
+ prevClose()
+ diskCleanupFn()
+ },
}
}
}
diff --git a/test/integration/standalone/multi_segments/property/suite_test.go b/test/integration/standalone/multi_segments/property/suite_test.go
index 88bbc050f..994c6f2e4 100644
--- a/test/integration/standalone/multi_segments/property/suite_test.go
+++ b/test/integration/standalone/multi_segments/property/suite_test.go
@@ -38,17 +38,34 @@ func init() {
Expect(tmpErr).NotTo(HaveOccurred())
dfWriter := setup.NewDiscoveryFileWriter(tmpDir)
config := setup.PropertyClusterConfig(dfWriter)
- addr, _, closeFn := setup.Standalone(config)
+ path, diskCleanupFn, pathErr := test.NewSpace()
+ Expect(pathErr).NotTo(HaveOccurred())
+ var ports []int
+ ports, portsErr := test.AllocateFreePorts(5)
+ Expect(portsErr).NotTo(HaveOccurred())
+ addr, _, closeFunc := setup.ClosableStandalone(config, path, ports)
ns := timestamp.NowMilli().UnixNano()
now := time.Unix(0, ns-ns%int64(time.Minute))
baseTime := time.Date(now.Year(), now.Month(), now.Day(), 0o0, 0o2, 0, 0, now.Location())
test_cases.Initialize(addr, baseTime)
+ prevClose, currClose := closeFunc, func() {}
return multisegments.SetupResult{
Addr: addr,
Now: now,
BaseTime: baseTime,
+ Restart: func() (string, func()) {
+ time.Sleep(5 * time.Second)
+ prevClose()
+ time.Sleep(3 * time.Second)
+ currClose()
+ addr, _, closeFunc := setup.EmptyClosableStandalone(config, path, ports)
+ prevClose, currClose = currClose, closeFunc
+ return addr, closeFunc
+ },
StopFunc: func() {
- closeFn()
+ currClose()
+ prevClose()
+ diskCleanupFn()
tmpDirCleanup()
},
}
diff --git a/test/integration/standalone/multi_segments/round2.go b/test/integration/standalone/multi_segments/round2.go
new file mode 100644
index 000000000..86cf80101
--- /dev/null
+++ b/test/integration/standalone/multi_segments/round2.go
@@ -0,0 +1,61 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Package multisegments provides shared test setup for multi-segment integration tests.
+package multisegments
+
+import (
+ "time"
+
+ "github.com/onsi/ginkgo/v2"
+ "github.com/onsi/gomega"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+
+ "github.com/apache/skywalking-banyandb/pkg/grpchelper"
+ "github.com/apache/skywalking-banyandb/pkg/test/helpers"
+ casesmeasure "github.com/apache/skywalking-banyandb/test/cases/measure"
+ casesstream "github.com/apache/skywalking-banyandb/test/cases/stream"
+ casestopn "github.com/apache/skywalking-banyandb/test/cases/topn"
+ casestrace "github.com/apache/skywalking-banyandb/test/cases/trace"
+)
+
+var round2Conn *grpc.ClientConn
+
+var _ = ginkgo.Describe("on-disk data", ginkgo.Ordered, func() {
+ ginkgo.BeforeAll(func() {
+ gomega.Expect(result.Restart).NotTo(gomega.BeNil())
+ newAddr, _ := result.Restart()
+ var connErr error
+ round2Conn, connErr = grpchelper.Conn(newAddr, 10*time.Second,
+ grpc.WithTransportCredentials(insecure.NewCredentials()))
+ gomega.Expect(connErr).NotTo(gomega.HaveOccurred())
+ sharedCtx := helpers.SharedContext{
+ Connection: round2Conn,
+ BaseTime: result.BaseTime,
+ }
+ casesstream.SharedContext = sharedCtx
+ casesmeasure.SharedContext = sharedCtx
+ casestopn.SharedContext = sharedCtx
+ casestrace.SharedContext = sharedCtx
+ })
+
+ casesstream.RegisterTable("Scanning Streams")
+ casesmeasure.RegisterTable("Scanning Measures")
+ casestopn.RegisterTable("TopN Tests")
+ casestrace.RegisterTable("Scanning Traces")
+})
diff --git a/test/integration/standalone/query/common.go b/test/integration/standalone/query/common.go
index 3d9eec194..49c44997c 100644
--- a/test/integration/standalone/query/common.go
+++ b/test/integration/standalone/query/common.go
@@ -42,6 +42,7 @@ import (
// SetupResult contains all info returned by SetupFunc.
type SetupResult struct {
+ Restart func() (string, func())
Now time.Time
StopFunc func()
Addr string
@@ -96,6 +97,9 @@ var _ = ginkgo.SynchronizedAfterSuite(func() {
if connection != nil {
gomega.Expect(connection.Close()).To(gomega.Succeed())
}
+ if round2Conn != nil {
+ gomega.Expect(round2Conn.Close()).To(gomega.Succeed())
+ }
}, func() {})
var _ = ginkgo.ReportAfterSuite("Integration Query Suite", func(report ginkgo.Report) {
diff --git a/test/integration/standalone/query/etcd/suite_test.go b/test/integration/standalone/query/etcd/suite_test.go
index fc8bbbdfc..7da983f55 100644
--- a/test/integration/standalone/query/etcd/suite_test.go
+++ b/test/integration/standalone/query/etcd/suite_test.go
@@ -24,6 +24,7 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
+ "github.com/apache/skywalking-banyandb/pkg/test"
"github.com/apache/skywalking-banyandb/pkg/test/setup"
"github.com/apache/skywalking-banyandb/pkg/timestamp"
test_cases "github.com/apache/skywalking-banyandb/test/cases"
@@ -33,14 +34,33 @@ import (
func init() {
query.SetupFunc = func() query.SetupResult {
- addr, _, cleanup := setup.Standalone(nil)
+ path, diskCleanupFn, pathErr := test.NewSpace()
+ Expect(pathErr).NotTo(HaveOccurred())
+ var ports []int
+ ports, portsErr := test.AllocateFreePorts(4)
+ Expect(portsErr).NotTo(HaveOccurred())
+ addr, _, closeFunc := setup.ClosableStandalone(nil, path, ports)
ns := timestamp.NowMilli().UnixNano()
now := time.Unix(0, ns-ns%int64(time.Minute))
test_cases.Initialize(addr, now)
+ prevClose, currClose := closeFunc, func() {}
return query.SetupResult{
- Addr: addr,
- Now: now,
- StopFunc: cleanup,
+ Addr: addr,
+ Now: now,
+ Restart: func() (string, func()) {
+ time.Sleep(5 * time.Second)
+ prevClose()
+ time.Sleep(3 * time.Second)
+ currClose()
+ addr, _, closeFunc := setup.EmptyClosableStandalone(nil, path, ports)
+ prevClose, currClose = currClose, closeFunc
+ return addr, closeFunc
+ },
+ StopFunc: func() {
+ currClose()
+ prevClose()
+ diskCleanupFn()
+ },
}
}
}
diff --git a/test/integration/standalone/query/property/suite_test.go b/test/integration/standalone/query/property/suite_test.go
index 41b5361b6..ac3b6b2c2 100644
--- a/test/integration/standalone/query/property/suite_test.go
+++ b/test/integration/standalone/query/property/suite_test.go
@@ -38,15 +38,32 @@ func init() {
Expect(tmpErr).NotTo(HaveOccurred())
dfWriter := setup.NewDiscoveryFileWriter(tmpDir)
config := setup.PropertyClusterConfig(dfWriter)
- addr, _, closeFn := setup.Standalone(config)
+ path, diskCleanupFn, pathErr := test.NewSpace()
+ Expect(pathErr).NotTo(HaveOccurred())
+ var ports []int
+ ports, portsErr := test.AllocateFreePorts(5)
+ Expect(portsErr).NotTo(HaveOccurred())
+ addr, _, closeFunc := setup.ClosableStandalone(config, path, ports)
ns := timestamp.NowMilli().UnixNano()
now := time.Unix(0, ns-ns%int64(time.Minute))
test_cases.Initialize(addr, now)
+ prevClose, currClose := closeFunc, func() {}
return query.SetupResult{
Addr: addr,
Now: now,
+ Restart: func() (string, func()) {
+ time.Sleep(5 * time.Second)
+ prevClose()
+ time.Sleep(3 * time.Second)
+ currClose()
+ addr, _, closeFunc := setup.EmptyClosableStandalone(config, path, ports)
+ prevClose, currClose = currClose, closeFunc
+ return addr, closeFunc
+ },
StopFunc: func() {
- closeFn()
+ currClose()
+ prevClose()
+ diskCleanupFn()
tmpDirCleanup()
},
}
diff --git a/test/integration/standalone/query/round2.go b/test/integration/standalone/query/round2.go
new file mode 100644
index 000000000..df76b1503
--- /dev/null
+++ b/test/integration/standalone/query/round2.go
@@ -0,0 +1,64 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Package query provides shared test setup for query integration tests.
+package query
+
+import (
+ "time"
+
+ "github.com/onsi/ginkgo/v2"
+ "github.com/onsi/gomega"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+
+ "github.com/apache/skywalking-banyandb/pkg/grpchelper"
+ "github.com/apache/skywalking-banyandb/pkg/test/helpers"
+ casesmeasure "github.com/apache/skywalking-banyandb/test/cases/measure"
+ casesproperty "github.com/apache/skywalking-banyandb/test/cases/property"
+ casesstream "github.com/apache/skywalking-banyandb/test/cases/stream"
+ casestopn "github.com/apache/skywalking-banyandb/test/cases/topn"
+ casestrace "github.com/apache/skywalking-banyandb/test/cases/trace"
+)
+
+var round2Conn *grpc.ClientConn
+
+var _ = ginkgo.Describe("on-disk data", ginkgo.Ordered, func() {
+ ginkgo.BeforeAll(func() {
+ gomega.Expect(result.Restart).NotTo(gomega.BeNil())
+ newAddr, _ := result.Restart()
+ var connErr error
+ round2Conn, connErr = grpchelper.Conn(newAddr, 10*time.Second,
+ grpc.WithTransportCredentials(insecure.NewCredentials()))
+ gomega.Expect(connErr).NotTo(gomega.HaveOccurred())
+ sharedCtx := helpers.SharedContext{
+ Connection: round2Conn,
+ BaseTime: result.Now,
+ }
+ casesstream.SharedContext = sharedCtx
+ casesmeasure.SharedContext = sharedCtx
+ casestopn.SharedContext = sharedCtx
+ casestrace.SharedContext = sharedCtx
+ casesproperty.SharedContext = sharedCtx
+ })
+
+ casesstream.RegisterTable("Scanning Streams")
+ casesmeasure.RegisterTable("Scanning Measures")
+ casestopn.RegisterTable("TopN Tests")
+ casestrace.RegisterTable("Scanning Traces")
+ casesproperty.RegisterTable("Scanning Properties")
+})
diff --git a/test/integration/standalone/query_ondisk/common.go b/test/integration/standalone/query_ondisk/common.go
deleted file mode 100644
index 455f6cd39..000000000
--- a/test/integration/standalone/query_ondisk/common.go
+++ /dev/null
@@ -1,103 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Package queryondisk provides shared test setup for on-disk query integration tests.
-package queryondisk
-
-import (
- "time"
-
- "github.com/onsi/ginkgo/v2"
- "github.com/onsi/gomega"
- "github.com/onsi/gomega/gleak"
- "google.golang.org/grpc"
- "google.golang.org/grpc/credentials/insecure"
-
- "github.com/apache/skywalking-banyandb/pkg/grpchelper"
- "github.com/apache/skywalking-banyandb/pkg/logger"
- "github.com/apache/skywalking-banyandb/pkg/pool"
- "github.com/apache/skywalking-banyandb/pkg/test/flags"
- "github.com/apache/skywalking-banyandb/pkg/test/gmatcher"
- "github.com/apache/skywalking-banyandb/pkg/test/helpers"
- casesmeasure "github.com/apache/skywalking-banyandb/test/cases/measure"
- casesstream "github.com/apache/skywalking-banyandb/test/cases/stream"
- casestopn "github.com/apache/skywalking-banyandb/test/cases/topn"
- casestrace "github.com/apache/skywalking-banyandb/test/cases/trace"
-)
-
-// SetupResult contains all info returned by SetupFunc.
-type SetupResult struct {
- Now time.Time
- StopFunc func()
- Addr string
-}
-
-// SetupFunc is provided by sub-packages to start the environment.
-var SetupFunc func() SetupResult
-
-var (
- result SetupResult
- connection *grpc.ClientConn
- goods []gleak.Goroutine
-)
-
-var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
- goods = gleak.Goroutines()
- gomega.Expect(logger.Init(logger.Logging{
- Env: "dev",
- Level: flags.LogLevel,
- })).To(gomega.Succeed())
- result = SetupFunc()
- return []byte(result.Addr)
-}, func(address []byte) {
- var err error
- connection, err = grpchelper.Conn(string(address), 10*time.Second,
- grpc.WithTransportCredentials(insecure.NewCredentials()))
- casesstream.SharedContext = helpers.SharedContext{
- Connection: connection,
- BaseTime: result.Now,
- }
- casesmeasure.SharedContext = helpers.SharedContext{
- Connection: connection,
- BaseTime: result.Now,
- }
- casestopn.SharedContext = helpers.SharedContext{
- Connection: connection,
- BaseTime: result.Now,
- }
- casestrace.SharedContext = helpers.SharedContext{
- Connection: connection,
- BaseTime: result.Now,
- }
- gomega.Expect(err).NotTo(gomega.HaveOccurred())
-})
-
-var _ = ginkgo.SynchronizedAfterSuite(func() {
- if connection != nil {
- gomega.Expect(connection.Close()).To(gomega.Succeed())
- }
-}, func() {})
-
-var _ = ginkgo.ReportAfterSuite("Integration Query OnDisk Suite", func(report ginkgo.Report) {
- if report.SuiteSucceeded {
- if result.StopFunc != nil {
- result.StopFunc()
- }
- gomega.Eventually(gleak.Goroutines, flags.EventuallyTimeout).ShouldNot(gleak.HaveLeaked(goods))
- gomega.Eventually(pool.AllRefsCount, flags.EventuallyTimeout).Should(gmatcher.HaveZeroRef())
- }
-})
diff --git a/test/integration/standalone/query_ondisk/etcd/suite_test.go b/test/integration/standalone/query_ondisk/etcd/suite_test.go
deleted file mode 100644
index e318c9b12..000000000
--- a/test/integration/standalone/query_ondisk/etcd/suite_test.go
+++ /dev/null
@@ -1,64 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package etcd_test
-
-import (
- "testing"
- "time"
-
- . "github.com/onsi/ginkgo/v2"
- . "github.com/onsi/gomega"
-
- "github.com/apache/skywalking-banyandb/pkg/test"
- "github.com/apache/skywalking-banyandb/pkg/test/setup"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
- test_cases "github.com/apache/skywalking-banyandb/test/cases"
- integration_standalone "github.com/apache/skywalking-banyandb/test/integration/standalone"
- "github.com/apache/skywalking-banyandb/test/integration/standalone/query_ondisk"
-)
-
-func init() {
- queryondisk.SetupFunc = func() queryondisk.SetupResult {
- path, diskCleanupFn, pathErr := test.NewSpace()
- Expect(pathErr).NotTo(HaveOccurred())
- var ports []int
- ports, portsErr := test.AllocateFreePorts(4)
- Expect(portsErr).NotTo(HaveOccurred())
- addr, _, closeFunc := setup.ClosableStandalone(nil, path, ports)
- ns := timestamp.NowMilli().UnixNano()
- now := time.Unix(0, ns-ns%int64(time.Minute))
- test_cases.Initialize(addr, now)
- time.Sleep(5 * time.Second)
- closeFunc()
- time.Sleep(time.Second)
- addr, _, closeFunc = setup.EmptyClosableStandalone(nil, path, ports)
- return queryondisk.SetupResult{
- Addr: addr,
- Now: now,
- StopFunc: func() {
- closeFunc()
- diskCleanupFn()
- },
- }
- }
-}
-
-func TestEtcdIntegrationQueryOnDisk(t *testing.T) {
- RegisterFailHandler(Fail)
- RunSpecs(t, "Integration Query OnDisk Suite", Label(integration_standalone.Labels...))
-}
diff --git a/test/integration/standalone/query_ondisk/property/suite_test.go b/test/integration/standalone/query_ondisk/property/suite_test.go
deleted file mode 100644
index 27a4f8cb7..000000000
--- a/test/integration/standalone/query_ondisk/property/suite_test.go
+++ /dev/null
@@ -1,69 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package property_test
-
-import (
- "testing"
- "time"
-
- . "github.com/onsi/ginkgo/v2"
- . "github.com/onsi/gomega"
-
- "github.com/apache/skywalking-banyandb/pkg/test"
- "github.com/apache/skywalking-banyandb/pkg/test/setup"
- "github.com/apache/skywalking-banyandb/pkg/timestamp"
- test_cases "github.com/apache/skywalking-banyandb/test/cases"
- integration_standalone "github.com/apache/skywalking-banyandb/test/integration/standalone"
- "github.com/apache/skywalking-banyandb/test/integration/standalone/query_ondisk"
-)
-
-func init() {
- queryondisk.SetupFunc = func() queryondisk.SetupResult {
- tmpDir, tmpDirCleanup, tmpErr := test.NewSpace()
- Expect(tmpErr).NotTo(HaveOccurred())
- dfWriter := setup.NewDiscoveryFileWriter(tmpDir)
- config := setup.PropertyClusterConfig(dfWriter)
- path, diskCleanupFn, pathErr := test.NewSpace()
- Expect(pathErr).NotTo(HaveOccurred())
- var ports []int
- ports, portsErr := test.AllocateFreePorts(5)
- Expect(portsErr).NotTo(HaveOccurred())
- addr, _, closeFunc := setup.ClosableStandalone(config, path, ports)
- ns := timestamp.NowMilli().UnixNano()
- now := time.Unix(0, ns-ns%int64(time.Minute))
- test_cases.Initialize(addr, now)
- time.Sleep(5 * time.Second)
- closeFunc()
- time.Sleep(time.Second)
- addr, _, closeFunc = setup.EmptyClosableStandalone(config, path, ports)
- return queryondisk.SetupResult{
- Addr: addr,
- Now: now,
- StopFunc: func() {
- closeFunc()
- diskCleanupFn()
- tmpDirCleanup()
- },
- }
- }
-}
-
-func TestPropertyIntegrationQueryOnDisk(t *testing.T) {
- RegisterFailHandler(Fail)
- RunSpecs(t, "Integration Query OnDisk Suite", Label(integration_standalone.Labels...))
-}
diff --git a/test_table b/test_table
new file mode 100755
index 000000000..5158e1c9b
Binary files /dev/null and b/test_table differ
(skywalking-banyandb) 11/14: Fix missing property group adapt when inspect group (#1117)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit e4e10d309ac897e71d85622dabade3d1bb6d6e14
Author: mrproliu <74...@qq.com>
AuthorDate: Thu May 7 15:34:35 2026 +0800
Fix missing property group adapt when inspect group (#1117)
---
CHANGES.md | 1 +
banyand/metadata/schema/collector.go | 18 +++++
banyand/metadata/schema/collector_test.go | 118 ++++++++++++++++++++++++++++++
3 files changed, 137 insertions(+)
diff --git a/CHANGES.md b/CHANGES.md
index 5cd522b15..22ba94f29 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -34,6 +34,7 @@ Release Notes.
- Fix trace `block_writer` panic on out-of-order timestamps within the same traceID, which dropped one trace-write batch per panic in multi-agent SkyWalking deployments. Spans of a single trace originate from independently-clocked services, and trace storage is organized by traceID rather than timestamp, so per-traceID timestamp monotonicity is not a writer invariant.
- Fix nil-pointer panic on cold-tier data nodes when FODC `InspectAll` raced with idle-segment cleanup.
- Add `GroupLifecycleInfo.errors` to surface per-group collection failures from FODC `InspectAll` instead of silently dropping the affected node entry.
+- Fix `CollectDataInfo` and `CollectLiaisonInfo` not handling `CATALOG_PROPERTY` groups.
### Chores
diff --git a/banyand/metadata/schema/collector.go b/banyand/metadata/schema/collector.go
index 6ece113de..27d198a71 100644
--- a/banyand/metadata/schema/collector.go
+++ b/banyand/metadata/schema/collector.go
@@ -99,6 +99,15 @@ func (icr *InfoCollectorRegistry) CollectDataInfo(ctx context.Context, group str
if getErr != nil {
return nil, nil, getErr
}
+ // PROPERTY groups are key-value metadata: no DataInfoCollector is
+ // registered for them and no Topic*CollectDataInfo broadcast applies.
+ // Skip the entire pipeline so the inspector reports nothing instead
+ // of falling into the unsupported-catalog default branch. Return a
+ // non-nil empty slice to match the shape of every other success path
+ // in this method (see localInfoList and broadcastCollectDataInfo).
+ if g.Catalog == commonv1.Catalog_CATALOG_PROPERTY {
+ return []*databasev1.DataInfo{}, nil, nil
+ }
localInfo, localErr := icr.collectDataInfoLocal(ctx, g.Catalog, group)
if localErr != nil {
return nil, nil, localErr
@@ -173,6 +182,15 @@ func (icr *InfoCollectorRegistry) CollectLiaisonInfo(ctx context.Context, group
if getErr != nil {
return nil, getErr
}
+ // PROPERTY groups are key-value metadata: no LiaisonInfoCollector is
+ // registered for them and no Topic*CollectLiaisonInfo broadcast applies.
+ // Skip the entire pipeline so the inspector reports nothing instead of
+ // falling into the unsupported-catalog default branch. Return a non-nil
+ // empty slice to match the shape of every other success path in this
+ // method (see localInfoList and broadcastCollectLiaisonInfo).
+ if g.Catalog == commonv1.Catalog_CATALOG_PROPERTY {
+ return []*databasev1.LiaisonInfo{}, nil
+ }
localInfo, localErr := icr.collectLiaisonInfoLocal(ctx, g.Catalog, group)
if localErr != nil {
return nil, localErr
diff --git a/banyand/metadata/schema/collector_test.go b/banyand/metadata/schema/collector_test.go
new file mode 100644
index 000000000..5ddf49732
--- /dev/null
+++ b/banyand/metadata/schema/collector_test.go
@@ -0,0 +1,118 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package schema_test
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+ "github.com/apache/skywalking-banyandb/banyand/metadata/schema"
+ "github.com/apache/skywalking-banyandb/pkg/bus"
+ "github.com/apache/skywalking-banyandb/pkg/logger"
+)
+
+type stubGroupGetter struct {
+ group *commonv1.Group
+ err error
+}
+
+func (s *stubGroupGetter) GetGroup(_ context.Context, _ string) (*commonv1.Group, error) {
+ if s.err != nil {
+ return nil, s.err
+ }
+ return s.group, nil
+}
+
+// failingBroadcaster fails the test if Broadcast is invoked. Used to assert
+// that catalogs without a lifecycle (e.g. CATALOG_PROPERTY) short-circuit
+// before reaching the broadcast path.
+type failingBroadcaster struct {
+ t *testing.T
+}
+
+func (f *failingBroadcaster) Broadcast(_ time.Duration, _ bus.Topic, _ bus.Message) ([]bus.Future, error) {
+ f.t.Helper()
+ f.t.Fatalf("Broadcast must not be called for catalogs without lifecycle inspection")
+ return nil, errors.New("unreachable")
+}
+
+func TestCollectDataInfo_PropertyCatalogShortCircuits(t *testing.T) {
+ registry := schema.NewInfoCollectorRegistry(logger.GetLogger("test"), &stubGroupGetter{
+ group: &commonv1.Group{
+ Metadata: &commonv1.Metadata{Name: "sw_property"},
+ Catalog: commonv1.Catalog_CATALOG_PROPERTY,
+ },
+ })
+ registry.SetDataBroadcaster(&failingBroadcaster{t: t})
+
+ dataInfo, collectionErrs, err := registry.CollectDataInfo(context.Background(), "sw_property")
+ require.NoError(t, err)
+ assert.Empty(t, dataInfo)
+ assert.Empty(t, collectionErrs)
+}
+
+func TestCollectLiaisonInfo_PropertyCatalogShortCircuits(t *testing.T) {
+ registry := schema.NewInfoCollectorRegistry(logger.GetLogger("test"), &stubGroupGetter{
+ group: &commonv1.Group{
+ Metadata: &commonv1.Metadata{Name: "_deletion_task"},
+ Catalog: commonv1.Catalog_CATALOG_PROPERTY,
+ },
+ })
+ registry.SetLiaisonBroadcaster(&failingBroadcaster{t: t})
+
+ liaisonInfo, err := registry.CollectLiaisonInfo(context.Background(), "_deletion_task")
+ require.NoError(t, err)
+ assert.Empty(t, liaisonInfo)
+}
+
+func TestCollectDataInfo_UnspecifiedCatalogStillReturnsError(t *testing.T) {
+ registry := schema.NewInfoCollectorRegistry(logger.GetLogger("test"), &stubGroupGetter{
+ group: &commonv1.Group{
+ Metadata: &commonv1.Metadata{Name: "weird_group"},
+ Catalog: commonv1.Catalog_CATALOG_UNSPECIFIED,
+ },
+ })
+ registry.SetDataBroadcaster(&failingBroadcaster{t: t})
+
+ dataInfo, collectionErrs, err := registry.CollectDataInfo(context.Background(), "weird_group")
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "unsupported catalog type")
+ assert.Nil(t, dataInfo)
+ assert.Nil(t, collectionErrs)
+}
+
+func TestCollectLiaisonInfo_UnspecifiedCatalogStillReturnsError(t *testing.T) {
+ registry := schema.NewInfoCollectorRegistry(logger.GetLogger("test"), &stubGroupGetter{
+ group: &commonv1.Group{
+ Metadata: &commonv1.Metadata{Name: "weird_group"},
+ Catalog: commonv1.Catalog_CATALOG_UNSPECIFIED,
+ },
+ })
+ registry.SetLiaisonBroadcaster(&failingBroadcaster{t: t})
+
+ liaisonInfo, err := registry.CollectLiaisonInfo(context.Background(), "weird_group")
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "unsupported catalog type")
+ assert.Nil(t, liaisonInfo)
+}
(skywalking-banyandb) 13/14: chore: fix lint issues after cherry-pick
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch v0.10.x
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 1f8e29f37c96fa81cb5e1fd0344bfc0a2750525f
Author: Hongtao Gao <ha...@gmail.com>
AuthorDate: Fri May 8 06:28:44 2026 +0000
chore: fix lint issues after cherry-pick
via [HAPI](https://hapi.run)
Co-Authored-By: HAPI <no...@hapi.run>
---
banyand/trace/metadata_test.go | 14 +++++---------
1 file changed, 5 insertions(+), 9 deletions(-)
diff --git a/banyand/trace/metadata_test.go b/banyand/trace/metadata_test.go
index 13c74a30e..58994f525 100644
--- a/banyand/trace/metadata_test.go
+++ b/banyand/trace/metadata_test.go
@@ -270,7 +270,7 @@ var _ = Describe("Metadata", func() {
Eventually(func(innerGm Gomega) {
spans := querySchemaChangeTraceData(svcs, traceName, groupName, now.Add(-3*time.Hour), now,
- []string{"trace_id", "service_id", "duration"}, nil)
+ []string{"trace_id", "service_id", "duration"})
innerGm.Expect(spans).To(HaveLen(8))
for _, span := range spans {
@@ -297,7 +297,7 @@ var _ = Describe("Metadata", func() {
Eventually(func(innerGm Gomega) {
spans := querySchemaChangeTraceData(svcs, traceName, groupName, now.Add(-3*time.Hour), now,
- []string{"trace_id", "service_id", "duration", "extra_tag"}, nil)
+ []string{"trace_id", "service_id", "duration", "extra_tag"})
innerGm.Expect(spans).To(HaveLen(8))
oldDataCount := 0
@@ -333,7 +333,7 @@ var _ = Describe("Metadata", func() {
Eventually(func(innerGm Gomega) {
spans := querySchemaChangeTraceData(svcs, traceName, groupName, now.Add(-3*time.Hour), now,
- []string{"trace_id", "service_id", "duration", "extra_tag"}, nil)
+ []string{"trace_id", "service_id", "duration", "extra_tag"})
innerGm.Expect(spans).To(HaveLen(8))
nullCount := 0
@@ -383,7 +383,7 @@ var _ = Describe("Metadata", func() {
Eventually(func(innerGm Gomega) {
spans := querySchemaChangeTraceData(svcs, traceName, groupName,
now.Add(-3*time.Hour), now,
- []string{"trace_id", "service_id", "duration", "extra_tag"}, nil)
+ []string{"trace_id", "service_id", "duration", "extra_tag"})
innerGm.Expect(spans).To(HaveLen(8))
nullCount := 0
@@ -704,7 +704,7 @@ func writeSchemaChangeTraceData(svcs *services, name, group string, baseTime tim
}
}
-func querySchemaChangeTraceData(svcs *services, name, group string, begin, end time.Time, tags []string, criteria *modelv1.Criteria) []*tracev1.Span {
+func querySchemaChangeTraceData(svcs *services, name, group string, begin, end time.Time, tags []string) []*tracev1.Span {
req := &tracev1.QueryRequest{
Groups: []string{group},
Name: name,
@@ -718,9 +718,6 @@ func querySchemaChangeTraceData(svcs *services, name, group string, begin, end t
Sort: modelv1.Sort_SORT_DESC,
},
}
- if criteria != nil {
- req.Criteria = criteria
- }
var spans []*tracev1.Span
Eventually(func() bool {
feat, err := svcs.pipeline.Publish(context.Background(), data.TopicTraceQuery, bus.NewMessage(bus.MessageID(time.Now().UnixNano()), req))
@@ -774,4 +771,3 @@ func getFilePartCount(svcs *services, group string) int64 {
}
return total
}
-