You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by zi...@apache.org on 2024/07/30 12:25:47 UTC
(pulsar-client-go) branch master updated: [fix] Avoid a data race when flushing with load (#1261)
This is an automated email from the ASF dual-hosted git repository.
zike pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar-client-go.git
The following commit(s) were added to refs/heads/master by this push:
new 8dd4ed19 [fix] Avoid a data race when flushing with load (#1261)
8dd4ed19 is described below
commit 8dd4ed19b98ecf9287cbcfc20e2de9a12b197e3f
Author: Gaylor Bosson <gb...@taurusgroup.ch>
AuthorDate: Tue Jul 30 14:25:42 2024 +0200
[fix] Avoid a data race when flushing with load (#1261)
Fixes #1258
### Motivation
While flushing, the data channel is switched if a new allocated one which can cause the loss of messages because the length can be zero which would stop the procedure and at the same time a new message can be sent to the channel.
### Modifications
Instead of allocating a new channel, it empties the existing one up to the length of the buffer of the channel before proceeding with the flush.
---
pulsar/producer_partition.go | 33 ++++++++++++++++++++-------------
1 file changed, 20 insertions(+), 13 deletions(-)
diff --git a/pulsar/producer_partition.go b/pulsar/producer_partition.go
index 5c038aa5..f5fd493b 100755
--- a/pulsar/producer_partition.go
+++ b/pulsar/producer_partition.go
@@ -338,10 +338,6 @@ func (p *partitionProducer) grabCnx(assignedBrokerURL string) error {
p.schemaCache.Put(p.schemaInfo, schemaVersion)
}
- if err != nil {
- return err
- }
-
if !p.options.DisableBatching && p.batchBuilder == nil {
provider, err := GetBatcherBuilderProvider(p.options.BatcherBuilderType)
if err != nil {
@@ -1022,15 +1018,7 @@ func (p *partitionProducer) internalFlushCurrentBatches() {
}
func (p *partitionProducer) internalFlush(fr *flushRequest) {
- // clear all the messages which have sent to dataChan before flush
- if len(p.dataChan) != 0 {
- oldDataChan := p.dataChan
- p.dataChan = make(chan *sendRequest, p.options.MaxPendingMessages)
- for len(oldDataChan) != 0 {
- pendingData := <-oldDataChan
- p.internalSend(pendingData)
- }
- }
+ p.clearPendingSendRequests()
if !p.options.DisableBatching {
p.internalFlushCurrentBatch()
@@ -1061,6 +1049,25 @@ func (p *partitionProducer) internalFlush(fr *flushRequest) {
}
}
+// clearPendingSendRequests makes sure to push forward previous sending requests
+// by emptying the data channel.
+func (p *partitionProducer) clearPendingSendRequests() {
+ sizeBeforeFlushing := len(p.dataChan)
+
+ // Bound the for loop to the current length of the channel to ensure that it
+ // will eventually stop as we only want to ensure that existing messages are
+ // flushed.
+ for i := 0; i < sizeBeforeFlushing; i++ {
+ select {
+ case pendingData := <-p.dataChan:
+ p.internalSend(pendingData)
+
+ default:
+ return
+ }
+ }
+}
+
func (p *partitionProducer) Send(ctx context.Context, msg *ProducerMessage) (MessageID, error) {
var err error
var msgID MessageID