Re: [PR] KAFKA-17108: Earliest pending upload offset [kafka]

["kamalcph (via GitHub)" <gi...@apache.org>]

kamalcph commented on code in PR #16584:
URL: https://github.com/apache/kafka/pull/16584#discussion_r1703789523


##########
core/src/main/scala/kafka/log/UnifiedLog.scala:
##########
@@ -1334,6 +1334,37 @@ class UnifiedLog(@volatile var logStartOffset: Long,
         } else {
           Some(new TimestampAndOffset(RecordBatch.NO_TIMESTAMP, -1L, Optional.of(-1)))
         }
+      } else if (targetTimestamp == ListOffsetsRequest.EARLIEST_PENDING_UPLOAD_OFFSET_TIMESTAMP) {
+        if (remoteLogEnabled()) {
+          val curHighestRemoteOffset = highestOffsetInRemoteStorage()
+
+          if (curHighestRemoteOffset == -1) {
+            if (localLogStartOffset() == logStartOffset) {
+              // No segments have been uploaded yet
+              fetchOffsetByTimestamp(ListOffsetsRequest.EARLIEST_TIMESTAMP, remoteLogManager)
+            } else {
+              // Leader currently does not know about the already uploaded segments
+              Some(new TimestampAndOffset(RecordBatch.NO_TIMESTAMP, -1L, Optional.of(-1)))
+            }
+          } else {
+            val earliestPendingUploadOffset = math.max(curHighestRemoteOffset + 1, logStartOffset)
+
+            val epochResult: Optional[Integer] =
+              if (leaderEpochCache.isDefined) {
+                val epochOpt = leaderEpochCache.get.epochForOffset(earliestPendingUploadOffset)
+                if (epochOpt.isPresent) {
+                  Optional.of(epochOpt.getAsInt)
+                } else {
+                  Optional.empty()
+                }
+              } else {
+                Optional.empty()
+              }
+            Some(new TimestampAndOffset(RecordBatch.NO_TIMESTAMP, earliestPendingUploadOffset, epochResult))

Review Comment:
   ```suggestion
               val epochResult = leaderEpochCache.flatMap(_.epochForOffset(earliestPendingUploadOffset).asScala)
                 .asJava.asInstanceOf[Optional[Integer]]
               Some(new TimestampAndOffset(RecordBatch.NO_TIMESTAMP, earliestPendingUploadOffset, epochResult))
   ```



##########
core/src/test/scala/unit/kafka/log/UnifiedLogTest.scala:
##########
@@ -2222,6 +2222,273 @@ class UnifiedLogTest {
       log.fetchOffsetByTimestamp(ListOffsetsRequest.LATEST_TIMESTAMP, Some(remoteLogManager)))
   }
 
+  @Test
+  def testFetchEarliestPendingUploadTimestampNoRemoteStorage(): Unit = {
+    val logConfig = LogTestUtils.createLogConfig(segmentBytes = 200, indexIntervalBytes = 1)
+    val log = createLog(logDir, logConfig)
+
+    assertEquals(Some(new TimestampAndOffset(ListOffsetsResponse.UNKNOWN_TIMESTAMP, -1, Optional.of(-1))),
+      log.fetchOffsetByTimestamp(ListOffsetsRequest.EARLIEST_PENDING_UPLOAD_OFFSET_TIMESTAMP))
+
+    val firstTimestamp = mockTime.milliseconds
+    val leaderEpoch = 0
+    log.appendAsLeader(TestUtils.singletonRecords(
+      value = TestUtils.randomBytes(10),
+      timestamp = firstTimestamp),
+      leaderEpoch = leaderEpoch)
+
+    val secondTimestamp = firstTimestamp + 1
+    log.appendAsLeader(TestUtils.singletonRecords(
+      value = TestUtils.randomBytes(10),
+      timestamp = secondTimestamp),
+      leaderEpoch = leaderEpoch)
+
+    assertEquals(Some(new TimestampAndOffset(ListOffsetsResponse.UNKNOWN_TIMESTAMP, -1, Optional.of(-1))),
+      log.fetchOffsetByTimestamp(ListOffsetsRequest.EARLIEST_PENDING_UPLOAD_OFFSET_TIMESTAMP))
+  }
+
+  @Test
+  def testFetchEarliestPendingUploadTimestampWithRemoteStorage(): Unit = {
+    val remoteLogManager = mock(classOf[RemoteLogManager])

Review Comment:
   we can refactor the test to extract the "prepare" phase which is duplicated across multiple tests. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org