You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2026/09/15 09:09:53 UTC

(skywalking-banyandb) 01/15: Fix standalone TopN write ordering (#1304)

This is an automated email from the ASF dual-hosted git repository.

hanahmily pushed a commit to branch release-0.11.1
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git

commit d5959ac1d3d1375bf9e4ea7b812a1b25f3caeed6
Author: Owen Willison <41...@users.noreply.github.com>
AuthorDate: Tue Aug 25 12:15:19 2026 +0800

    Fix standalone TopN write ordering (#1304)
    
    (cherry picked from commit cf3b770b9aeb85c4ff883723bcb9f646a494be6b)
---
 banyand/measure/topn.go                 | 190 ++++++++++++++++++++++++--------
 banyand/measure/topn_benchmark_test.go  | 140 +++++++++++++++++++++++
 banyand/measure/topn_unregister_test.go | 172 ++++++++++++++++++++++++++++-
 3 files changed, 453 insertions(+), 49 deletions(-)

diff --git a/banyand/measure/topn.go b/banyand/measure/topn.go
index bda30bf52..ad470ce76 100644
--- a/banyand/measure/topn.go
+++ b/banyand/measure/topn.go
@@ -59,6 +59,7 @@ const (
 	resultPersistencyTimeout = 10 * time.Second
 	maxFlushInterval         = time.Minute
 	maxTopNValuesCount       = 1 << 17 // sanity upper bound: 131072 elements = ~1MB per unmarshal, far above TopN CountersNumber
+	topNWriteQueueCapacity   = 1024
 )
 
 var (
@@ -333,7 +334,7 @@ type topNProcessor interface {
 	Setup(ctx context.Context) error
 	Teardown(ctx context.Context) error
 	Close() error
-	Src() chan interface{}
+	Write(flow.StreamRecord)
 	TopNSchema() *databasev1.TopNAggregation
 }
 
@@ -341,16 +342,20 @@ type topNStreamingProcessor[K streaming.TopSortKey] struct {
 	pipeline      queue.Client
 	streamingFlow flow.Flow
 	in            chan flow.StreamRecord
-	l             *logger.Logger
-	topNSchema    *databasev1.TopNAggregation
-	src           chan interface{}
+	writeStopCh   chan struct{}
 	m             *databasev1.Measure
 	errCh         <-chan error
+	topNSchema    *databasev1.TopNAggregation
+	l             *logger.Logger
 	stopCh        chan struct{}
+	src           chan interface{}
 	nodeID        string
 	flow.ComponentState
 	interval      time.Duration
+	closeOnce     sync.Once
+	writeMu       sync.Mutex
 	sortDirection modelv1.Sort
+	closed        bool
 }
 
 func newTopNStreamingProcessor[K streaming.TopSortKey](
@@ -373,6 +378,7 @@ func newTopNStreamingProcessor[K streaming.TopSortKey](
 		src:           srcCh,
 		in:            make(chan flow.StreamRecord),
 		stopCh:        make(chan struct{}),
+		writeStopCh:   make(chan struct{}),
 		streamingFlow: streamingFlow,
 		pipeline:      pipeline,
 		nodeID:        nodeID,
@@ -418,7 +424,17 @@ func (t *topNStreamingProcessor[K]) Teardown(_ context.Context) error {
 }
 
 func (t *topNStreamingProcessor[K]) Close() error {
+	t.closeOnce.Do(func() {
+		close(t.writeStopCh)
+	})
+	t.writeMu.Lock()
+	if t.closed {
+		t.writeMu.Unlock()
+		return nil
+	}
+	t.closed = true
 	close(t.src)
+	t.writeMu.Unlock()
 	// close streaming flow
 	err := t.streamingFlow.Close()
 	// and wait for error channel close
@@ -427,8 +443,16 @@ func (t *topNStreamingProcessor[K]) Close() error {
 	return err
 }
 
-func (t *topNStreamingProcessor[K]) Src() chan interface{} {
-	return t.src
+func (t *topNStreamingProcessor[K]) Write(record flow.StreamRecord) {
+	t.writeMu.Lock()
+	defer t.writeMu.Unlock()
+	if t.closed {
+		return
+	}
+	select {
+	case t.src <- record:
+	case <-t.writeStopCh:
+	}
 }
 
 func (t *topNStreamingProcessor[K]) TopNSchema() *databasev1.TopNAggregation {
@@ -596,16 +620,27 @@ func (t *topNStreamingProcessor[K]) handleError() {
 	close(t.stopCh)
 }
 
+type topNWrite struct {
+	request  *measurev1.InternalWriteRequest
+	measure  *databasev1.Measure
+	seriesID uint64
+	shardID  uint32
+}
+
 // topNProcessorManager manages multiple topNStreamingProcessor(s) belonging to a single measure.
 type topNProcessorManager struct {
 	pipeline        queue.Client
 	l               *logger.Logger
 	m               *databasev1.Measure
+	inputCh         chan topNWrite
+	stopCh          chan struct{}
+	dispatcher      *run.Task
 	nodeID          string
 	registeredTasks []*databasev1.TopNAggregation
 	processorList   []topNProcessor
 	sync.RWMutex
-	closed bool
+	enqueueMu sync.Mutex
+	closed    bool
 }
 
 func (manager *topNProcessorManager) init(ctx context.Context, m *databasev1.Measure) {
@@ -614,6 +649,11 @@ func (manager *topNProcessorManager) init(ctx context.Context, m *databasev1.Mea
 	if manager.closed {
 		return
 	}
+	if manager.inputCh == nil {
+		manager.inputCh = make(chan topNWrite, topNWriteQueueCapacity)
+		manager.stopCh = make(chan struct{})
+		manager.dispatcher = run.Go(ctx, "measure.topn.dispatcher", manager.l, manager.dispatchTopNWrites)
+	}
 	if manager.m != nil {
 		return
 	}
@@ -627,26 +667,36 @@ func (manager *topNProcessorManager) init(ctx context.Context, m *databasev1.Mea
 
 func (manager *topNProcessorManager) Close() error {
 	manager.Lock()
-	defer manager.Unlock()
 	if manager.closed {
+		manager.Unlock()
 		return nil
 	}
 	manager.closed = true
+	processorList := manager.processorList
+	manager.processorList = nil
+	manager.registeredTasks = nil
+	manager.m = nil
+	stopCh := manager.stopCh
+	dispatcher := manager.dispatcher
+	manager.Unlock()
+	if stopCh != nil {
+		close(stopCh)
+	}
 	// Close all processors in parallel to avoid serial 5-second-per-flow timeouts.
-	errCh := make(chan error, len(manager.processorList))
-	for _, processor := range manager.processorList {
+	errCh := make(chan error, len(processorList))
+	for _, processor := range processorList {
 		p := processor
 		run.Go(context.Background(), "measure.topn.processor-close", manager.l, func(_ context.Context) {
 			errCh <- p.Close()
 		})
 	}
 	var err error
-	for range manager.processorList {
+	for range processorList {
 		err = multierr.Append(err, <-errCh)
 	}
-	manager.processorList = nil
-	manager.registeredTasks = nil
-	manager.m = nil
+	if dispatcher != nil {
+		dispatcher.Wait()
+	}
 	return err
 }
 
@@ -657,66 +707,106 @@ func (manager *topNProcessorManager) onMeasureWrite(
 	request *measurev1.InternalWriteRequest,
 	measure *databasev1.Measure,
 ) {
-	run.Go(ctx, "topn-write", manager.l, func(_ context.Context) {
+	manager.enqueueMu.Lock()
+	defer manager.enqueueMu.Unlock()
+	manager.RLock()
+	if manager.closed {
+		manager.RUnlock()
+		return
+	}
+	if manager.m == nil {
+		manager.RUnlock()
+		manager.init(ctx, measure)
 		manager.RLock()
-		defer manager.RUnlock()
-		if manager.closed {
+		if manager.closed || manager.m == nil || manager.inputCh == nil {
+			manager.RUnlock()
 			return
 		}
-		if manager.m == nil {
-			manager.RUnlock()
-			manager.init(ctx, measure)
-			manager.RLock()
+	}
+	inputCh := manager.inputCh
+	stopCh := manager.stopCh
+	manager.RUnlock()
+	input := topNWrite{
+		seriesID: seriesID,
+		shardID:  shardID,
+		request:  request,
+		measure:  measure,
+	}
+	select {
+	case inputCh <- input:
+	case <-stopCh:
+	}
+}
+
+func (manager *topNProcessorManager) dispatchTopNWrites(_ context.Context) {
+	for {
+		select {
+		case <-manager.stopCh:
+			return
+		default:
 		}
-		dp := request.GetRequest().GetDataPoint()
-		spec := request.GetRequest().GetDataPointSpec()
-		for _, processor := range manager.processorList {
-			dpWithEntity := newDataPointWithEntityValues(
-				dp,
-				request.GetEntityValues(),
-				seriesID,
-				shardID,
-				spec,
-				manager.m,
-			)
-			processor.Src() <- flow.NewStreamRecordWithTimestampPb(dpWithEntity, dp.GetTimestamp())
+		select {
+		case input := <-manager.inputCh:
+			manager.dispatchTopNWrite(input)
+		case <-manager.stopCh:
+			return
 		}
-	})
+	}
+}
+
+func (manager *topNProcessorManager) dispatchTopNWrite(input topNWrite) {
+	manager.RLock()
+	if manager.closed {
+		manager.RUnlock()
+		return
+	}
+	processors := slices.Clone(manager.processorList)
+	manager.RUnlock()
+	dp := input.request.GetRequest().GetDataPoint()
+	spec := input.request.GetRequest().GetDataPointSpec()
+	for _, processor := range processors {
+		dpWithEntity := newDataPointWithEntityValues(
+			dp,
+			input.request.GetEntityValues(),
+			input.seriesID,
+			input.shardID,
+			spec,
+			input.measure,
+		)
+		processor.Write(flow.NewStreamRecordWithTimestampPb(dpWithEntity, dp.GetTimestamp()))
+	}
 }
 
 func (manager *topNProcessorManager) register(ctx context.Context, topNSchema *databasev1.TopNAggregation) {
 	manager.Lock()
-	defer manager.Unlock()
 	if manager.closed {
+		manager.Unlock()
 		return
 	}
-	exist := false
 	for i := range manager.registeredTasks {
 		if manager.registeredTasks[i].GetMetadata().GetName() == topNSchema.GetMetadata().GetName() {
-			exist = true
 			if manager.registeredTasks[i].GetMetadata().GetModRevision() < topNSchema.GetMetadata().GetModRevision() {
 				prev := manager.registeredTasks[i]
 				prevProcessors := manager.removeProcessors(prev)
 				if err := manager.start(ctx, topNSchema); err != nil {
 					manager.l.Err(err).Msg("fail to start the new processor")
+					manager.Unlock()
 					return
 				}
 				manager.registeredTasks[i] = topNSchema
-				for _, processor := range prevProcessors {
-					if err := processor.Close(); err != nil {
-						manager.l.Err(err).Msg("fail to close the prev processor")
-					}
-				}
+				manager.Unlock()
+				manager.closeProcessors(prevProcessors, "fail to close the prev processor")
+				return
 			}
+			manager.Unlock()
+			return
 		}
 	}
-	if exist {
-		return
-	}
 	manager.registeredTasks = append(manager.registeredTasks, topNSchema)
 	if err := manager.start(ctx, topNSchema); err != nil {
 		manager.l.Err(err).Msg("fail to start processor")
 	}
+	manager.Unlock()
 }
 
 func (manager *topNProcessorManager) start(ctx context.Context, topNSchema *databasev1.TopNAggregation) error {
@@ -784,12 +874,16 @@ func (manager *topNProcessorManager) unregister(topNSchema *databasev1.TopNAggre
 	manager.Unlock()
 	// Close outside the lock: processor.Close can block up to ~5s and would otherwise
 	// stall onMeasureWrite readers; the processors are already detached from the manager.
-	for _, processor := range removed {
+	manager.closeProcessors(removed, "fail to close the removed top-n processor")
+	return empty
+}
+
+func (manager *topNProcessorManager) closeProcessors(processors []topNProcessor, message string) {
+	for _, processor := range processors {
 		if err := processor.Close(); err != nil {
-			manager.l.Err(err).Msg("fail to close the removed top-n processor")
+			manager.l.Err(err).Msg(message)
 		}
 	}
-	return empty
 }
 
 func (manager *topNProcessorManager) removeProcessors(topNSchema *databasev1.TopNAggregation) []topNProcessor {
diff --git a/banyand/measure/topn_benchmark_test.go b/banyand/measure/topn_benchmark_test.go
new file mode 100644
index 000000000..b09ec9284
--- /dev/null
+++ b/banyand/measure/topn_benchmark_test.go
@@ -0,0 +1,140 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with this work for
+// additional information regarding copyright ownership. The ASF licenses this
+// file to You under the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License. You may obtain a
+// copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package measure
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
+	databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+	measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
+	"github.com/apache/skywalking-banyandb/pkg/flow"
+	"github.com/apache/skywalking-banyandb/pkg/run"
+)
+
+const topNWriteBenchmarkBatchSize = 256
+
+type benchmarkTopNProcessor struct {
+	completed *sync.WaitGroup
+}
+
+func (b *benchmarkTopNProcessor) In() chan<- flow.StreamRecord            { return nil }
+func (b *benchmarkTopNProcessor) Setup(context.Context) error             { return nil }
+func (b *benchmarkTopNProcessor) Teardown(context.Context) error          { return nil }
+func (b *benchmarkTopNProcessor) Close() error                            { return nil }
+func (b *benchmarkTopNProcessor) TopNSchema() *databasev1.TopNAggregation { return nil }
+
+func (b *benchmarkTopNProcessor) Write(flow.StreamRecord) {
+	b.completed.Done()
+}
+
+// BenchmarkTopNWriteDispatch compares completed write throughput. Each batch waits
+// for every record to reach the processor, so the asynchronous baseline cannot
+// report only enqueue latency while leaving its goroutines outstanding.
+func BenchmarkTopNWriteDispatch(b *testing.B) {
+	b.Run("original_async", func(b *testing.B) {
+		benchmarkOriginalAsyncTopNWrites(b)
+	})
+	b.Run("bounded_dispatcher", func(b *testing.B) {
+		benchmarkBoundedTopNWrites(b)
+	})
+}
+
+func benchmarkOriginalAsyncTopNWrites(b *testing.B) {
+	completed := &sync.WaitGroup{}
+	processor := &benchmarkTopNProcessor{completed: completed}
+	measureSchema := benchmarkTopNMeasureSchema()
+	manager := &topNProcessorManager{
+		m:             measureSchema,
+		processorList: []topNProcessor{processor},
+	}
+	request := benchmarkTopNWriteRequest()
+	b.ReportAllocs()
+	b.ResetTimer()
+	benchmarkTopNWrites(b, completed, func() {
+		benchmarkOriginalAsyncTopNWrite(manager, request, measureSchema)
+	})
+}
+
+func benchmarkBoundedTopNWrites(b *testing.B) {
+	completed := &sync.WaitGroup{}
+	processor := &benchmarkTopNProcessor{completed: completed}
+	measureSchema := benchmarkTopNMeasureSchema()
+	manager := &topNProcessorManager{processorList: []topNProcessor{processor}}
+	manager.init(context.Background(), measureSchema)
+	b.Cleanup(func() {
+		if closeErr := manager.Close(); closeErr != nil {
+			b.Fatal(closeErr)
+		}
+	})
+	request := benchmarkTopNWriteRequest()
+	b.ReportAllocs()
+	b.ResetTimer()
+	benchmarkTopNWrites(b, completed, func() {
+		manager.onMeasureWrite(context.Background(), 1, 1, request, measureSchema)
+	})
+}
+
+func benchmarkTopNWrites(b *testing.B, completed *sync.WaitGroup, write func()) {
+	for writesRemaining := b.N; writesRemaining > 0; {
+		batchSize := min(writesRemaining, topNWriteBenchmarkBatchSize)
+		completed.Add(batchSize)
+		for range batchSize {
+			write()
+		}
+		completed.Wait()
+		writesRemaining -= batchSize
+	}
+}
+
+func benchmarkOriginalAsyncTopNWrite(manager *topNProcessorManager, request *measurev1.InternalWriteRequest, measureSchema *databasev1.Measure) {
+	run.Go(context.Background(), "measure.topn.benchmark-original-async", nil, func(_ context.Context) {
+		manager.RLock()
+		defer manager.RUnlock()
+		dp := request.GetRequest().GetDataPoint()
+		spec := request.GetRequest().GetDataPointSpec()
+		for _, processor := range manager.processorList {
+			dpWithEntity := newDataPointWithEntityValues(
+				dp,
+				request.GetEntityValues(),
+				1,
+				1,
+				spec,
+				measureSchema,
+			)
+			processor.Write(flow.NewStreamRecordWithTimestampPb(dpWithEntity, dp.GetTimestamp()))
+		}
+	})
+}
+
+func benchmarkTopNMeasureSchema() *databasev1.Measure {
+	return &databasev1.Measure{Metadata: &commonv1.Metadata{Group: "benchmark", Name: "topn"}}
+}
+
+func benchmarkTopNWriteRequest() *measurev1.InternalWriteRequest {
+	return &measurev1.InternalWriteRequest{
+		Request: &measurev1.WriteRequest{
+			DataPoint: &measurev1.DataPointValue{
+				Timestamp: timestamppb.New(time.UnixMilli(1_000)),
+			},
+		},
+	}
+}
diff --git a/banyand/measure/topn_unregister_test.go b/banyand/measure/topn_unregister_test.go
index 8cbdc2a05..bbdce5a64 100644
--- a/banyand/measure/topn_unregister_test.go
+++ b/banyand/measure/topn_unregister_test.go
@@ -19,12 +19,16 @@ package measure
 
 import (
 	"context"
+	"sync"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/types/known/timestamppb"
 
 	commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
 	databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+	measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
 	"github.com/apache/skywalking-banyandb/pkg/flow"
 	"github.com/apache/skywalking-banyandb/pkg/logger"
 )
@@ -38,9 +42,46 @@ func (f fakeTopNProcessor) In() chan<- flow.StreamRecord            { return nil
 func (f fakeTopNProcessor) Setup(context.Context) error             { return nil }
 func (f fakeTopNProcessor) Teardown(context.Context) error          { return nil }
 func (f fakeTopNProcessor) Close() error                            { *f.closed = true; return nil }
-func (f fakeTopNProcessor) Src() chan interface{}                   { return nil }
+func (f fakeTopNProcessor) Write(flow.StreamRecord)                 {}
 func (f fakeTopNProcessor) TopNSchema() *databasev1.TopNAggregation { return f.schema }
 
+type blockingTopNProcessor struct {
+	writeStarted chan struct{}
+	closeStarted chan struct{}
+	releaseWrite chan struct{}
+	writeMu      sync.Mutex
+	closed       bool
+}
+
+func (b *blockingTopNProcessor) In() chan<- flow.StreamRecord            { return nil }
+func (b *blockingTopNProcessor) Setup(context.Context) error             { return nil }
+func (b *blockingTopNProcessor) Teardown(context.Context) error          { return nil }
+func (b *blockingTopNProcessor) TopNSchema() *databasev1.TopNAggregation { return nil }
+
+func (b *blockingTopNProcessor) Write(flow.StreamRecord) {
+	b.writeMu.Lock()
+	defer b.writeMu.Unlock()
+	close(b.writeStarted)
+	<-b.releaseWrite
+}
+
+func (b *blockingTopNProcessor) Close() error {
+	close(b.closeStarted)
+	b.writeMu.Lock()
+	b.closed = true
+	b.writeMu.Unlock()
+	return nil
+}
+
+type recordingTopNProcessor struct {
+	fakeTopNProcessor
+	records chan flow.StreamRecord
+}
+
+func (r recordingTopNProcessor) Write(record flow.StreamRecord) {
+	r.records <- record
+}
+
 func agg(name string) *databasev1.TopNAggregation {
 	return &databasev1.TopNAggregation{Metadata: &commonv1.Metadata{Group: "g", Name: name}}
 }
@@ -103,3 +144,132 @@ func TestRemoveTopNAggregation_DropsManagerOnlyWhenEmpty(t *testing.T) {
 	require.False(t, ok, "manager dropped from the map after the last aggregation is removed")
 	require.True(t, closedB)
 }
+
+// TestTopNManager_CloseDoesNotHoldManagerLockWhileWriterDrains ensures a blocked
+// processor write cannot prevent the manager from transitioning to closed.
+func TestTopNManager_CloseDoesNotHoldManagerLockWhileWriterDrains(t *testing.T) {
+	processor := &blockingTopNProcessor{
+		writeStarted: make(chan struct{}),
+		closeStarted: make(chan struct{}),
+		releaseWrite: make(chan struct{}),
+	}
+	measureSchema := &databasev1.Measure{}
+	manager := &topNProcessorManager{
+		processorList: []topNProcessor{processor},
+	}
+	manager.init(context.Background(), measureSchema)
+	request := &measurev1.InternalWriteRequest{
+		Request: &measurev1.WriteRequest{DataPoint: &measurev1.DataPointValue{}},
+	}
+	writeDone := make(chan struct{})
+	go func() {
+		manager.onMeasureWrite(context.Background(), 0, 0, request, measureSchema)
+		close(writeDone)
+	}()
+	require.Eventually(t, func() bool {
+		select {
+		case <-processor.writeStarted:
+			return true
+		default:
+			return false
+		}
+	}, time.Second, time.Millisecond)
+
+	closeErr := make(chan error, 1)
+	go func() {
+		closeErr <- manager.Close()
+	}()
+	require.Eventually(t, func() bool {
+		select {
+		case <-processor.closeStarted:
+			return true
+		default:
+			return false
+		}
+	}, time.Second, time.Millisecond)
+
+	managerClosed := make(chan bool, 1)
+	go func() {
+		manager.Lock()
+		closed := manager.closed
+		manager.Unlock()
+		managerClosed <- closed
+	}()
+	require.Eventually(t, func() bool {
+		select {
+		case isClosed := <-managerClosed:
+			return isClosed
+		default:
+			return false
+		}
+	}, time.Second, time.Millisecond)
+
+	close(processor.releaseWrite)
+	require.Eventually(t, func() bool {
+		select {
+		case <-writeDone:
+			return true
+		default:
+			return false
+		}
+	}, time.Second, time.Millisecond)
+	require.Eventually(t, func() bool {
+		select {
+		case closeResult := <-closeErr:
+			require.NoError(t, closeResult)
+			return true
+		default:
+			return false
+		}
+	}, time.Second, time.Millisecond)
+}
+
+func TestTopNManagerQueuesWritesInOrder(t *testing.T) {
+	const (
+		firstTimestamp  = 1_000
+		secondTimestamp = 61_000
+	)
+	measureSchema := &databasev1.Measure{Metadata: &commonv1.Metadata{Group: "g", Name: "m"}}
+	var closed bool
+	records := make(chan flow.StreamRecord, 2)
+	manager := &topNProcessorManager{
+		l: logger.GetLogger("test"),
+		processorList: []topNProcessor{
+			recordingTopNProcessor{
+				fakeTopNProcessor: fakeTopNProcessor{schema: agg("a"), closed: &closed},
+				records:           records,
+			},
+		},
+	}
+	manager.init(context.Background(), measureSchema)
+	t.Cleanup(func() {
+		require.NoError(t, manager.Close())
+	})
+	manager.onMeasureWrite(context.Background(), 1, 0, topNWriteRequest(firstTimestamp), measureSchema)
+	manager.onMeasureWrite(context.Background(), 2, 0, topNWriteRequest(secondTimestamp), measureSchema)
+	firstRecord := waitForTopNRecord(t, records)
+	secondRecord := waitForTopNRecord(t, records)
+	require.Equal(t, int64(firstTimestamp), firstRecord.TimestampMillis())
+	require.Equal(t, int64(secondTimestamp), secondRecord.TimestampMillis())
+}
+
+func waitForTopNRecord(t *testing.T, records <-chan flow.StreamRecord) flow.StreamRecord {
+	t.Helper()
+	select {
+	case record := <-records:
+		return record
+	case <-time.After(time.Second):
+		require.FailNow(t, "timed out waiting for a queued TopN write")
+		return flow.StreamRecord{}
+	}
+}
+
+func topNWriteRequest(timestampMillis int64) *measurev1.InternalWriteRequest {
+	return &measurev1.InternalWriteRequest{
+		Request: &measurev1.WriteRequest{
+			DataPoint: &measurev1.DataPointValue{
+				Timestamp: timestamppb.New(time.UnixMilli(timestampMillis)),
+			},
+		},
+	}
+}