[GitHub] [pulsar-client-go] cckellogg commented on a change in pull request #696: Add parition reader to solve the Reader fragmented read data.

[GitBox <gi...@apache.org>]

cckellogg commented on a change in pull request #696:
URL: https://github.com/apache/pulsar-client-go/pull/696#discussion_r776832958



##########
File path: pulsar/reader_impl.go
##########
@@ -148,35 +113,54 @@ func (r *reader) HasNext() bool {
 		return true
 	}
 
+retryLoop:
 	for {
-		lastMsgID, err := r.pc.getLastMessageID()
-		if err != nil {
-			r.log.WithError(err).Error("Failed to get last message id from broker")
-			continue
-		} else {
+	consumerLoop:
+		for _, consumer := range r.consumers {
+			lastMsgID, err := consumer.getLastMessageID()
+			if err != nil {
+				r.log.WithError(err).Error("Failed to get last message id from broker")
+				continue retryLoop
+			}
+			if r.lastMessageInBroker.greater(lastMsgID.messageID) {
+				continue consumerLoop
+			}
 			r.lastMessageInBroker = lastMsgID
-			break
 		}
+		break retryLoop
 	}
 
 	return r.hasMoreMessages()
 }
 
 func (r *reader) hasMoreMessages() bool {
-	if !r.pc.lastDequeuedMsg.Undefined() {
-		return r.lastMessageInBroker.isEntryIDValid() && r.lastMessageInBroker.greater(r.pc.lastDequeuedMsg.messageID)
-	}
+	moreMessagesCheck := func(idx int) bool {
+		if !r.consumers[idx].lastDequeuedMsg.Undefined() {
+			return r.lastMessageInBroker.isEntryIDValid() && r.lastMessageInBroker.greater(r.consumers[idx].lastDequeuedMsg.messageID)
+		}
 
-	if r.pc.options.startMessageIDInclusive {
-		return r.lastMessageInBroker.isEntryIDValid() && r.lastMessageInBroker.greaterEqual(r.pc.startMessageID.messageID)
-	}
+		if r.consumers[idx].options.startMessageIDInclusive {
+			return r.lastMessageInBroker.isEntryIDValid() && r.lastMessageInBroker.greaterEqual(r.consumers[idx].startMessageID.messageID)
+		}
 
-	// Non-inclusive
-	return r.lastMessageInBroker.isEntryIDValid() && r.lastMessageInBroker.greater(r.pc.startMessageID.messageID)
+		// Non-inclusive
+		return r.lastMessageInBroker.isEntryIDValid() && r.lastMessageInBroker.greater(r.consumers[idx].startMessageID.messageID)
+	}
+	for idx := range r.consumers {
+		if moreMessagesCheck(idx) {

Review comment:
       Let's pass the consumer and the lastMessageInBroker (instead of an index) `consumerHasMoreMessages(consumer Consumer, lastMessageInBroker)` Then the logic can be Tested.

##########
File path: pulsar/reader_partition.go
##########
@@ -0,0 +1,142 @@
+package pulsar
+
+import (
+	"sync"
+	"time"
+)
+
+func (r *reader) internalTopicReadToPartitions() error {

Review comment:
       What is this function doing I'm a little confused by the name. Also, this looks like a lot of duplication code from the consumer partitions is there any way to refactor/combine in to helper functions to avoid duplication?

##########
File path: pulsar/reader_impl.go
##########
@@ -207,12 +191,15 @@ func (r *reader) Seek(msgID MessageID) error {
 		return nil
 	}
 
-	return r.pc.Seek(mid)
+	return r.consumers[mid.partitionIdx].Seek(mid)

Review comment:
       could there be an invalid partition (out of range) index here?




-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@pulsar.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org