You are viewing a plain text version of this content. The canonical link for it is here.
Posted to jira@kafka.apache.org by "showuon (via GitHub)" <gi...@apache.org> on 2023/09/08 09:27:46 UTC
[GitHub] [kafka] showuon commented on a diff in pull request #14243: KAFKA-14912:Add a dynamic config for remote index cache size
showuon commented on code in PR #14243:
URL: https://github.com/apache/kafka/pull/14243#discussion_r1319305787
##########
core/src/test/scala/unit/kafka/log/remote/RemoteIndexCacheTest.scala:
##########
@@ -171,21 +178,24 @@ class RemoteIndexCacheTest {
// Calling getIndex on the same entry should not call rsm#fetchIndex again, but it should retrieve from cache
cache.getIndexEntry(metadataList.head)
assertCacheSize(1)
- verifyFetchIndexInvocation(count = 1)
+ // contains the 1 time for rsm when estimating entry size
+ verifyFetchIndexInvocation(count = 2)
// Here a new key metadataList(1) is invoked, that should call rsm#fetchIndex, making the count to 2
cache.getIndexEntry(metadataList.head)
cache.getIndexEntry(metadataList(1))
assertCacheSize(2)
- verifyFetchIndexInvocation(count = 2)
+ // contains the 1 time for rsm when estimating entry size
+ verifyFetchIndexInvocation(count = 3)
Review Comment:
The comment in L184 should be updated.
##########
core/src/test/scala/unit/kafka/log/remote/RemoteIndexCacheTest.scala:
##########
@@ -159,9 +159,16 @@ class RemoteIndexCacheTest {
@Test
def testCacheEntryExpiry(): Unit = {
// close existing cache created in test setup before creating a new one
+ val cacheForEstimate = new RemoteIndexCache(2L, rsm, tpDir.toString)
+ val tpIdForEstimate = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0))
+ val metadataListForEstimate = generateRemoteLogSegmentMetadata(size = 1, tpIdForEstimate)
+ val entryForEstimate = cacheForEstimate.getIndexEntry(metadataListForEstimate.head)
+ val estimateEntryBytesSize = entryForEstimate.entrySizeBytes()
+ Utils.closeQuietly(cacheForEstimate, "RemoteIndexCache created for estimating entry size")
Review Comment:
This estimation should only do once for this test suite. Could we move them into `beforeAll` ?
##########
storage/src/main/java/org/apache/kafka/storage/internals/log/RemoteIndexCache.java:
##########
@@ -494,6 +518,19 @@ public boolean isMarkedForCleanup() {
return markedForCleanup;
}
+ public long entrySizeBytes() {
+ return entrySizeBytes;
+ }
+
+ public long estimatedEntrySize() {
+ try {
+ return Files.size(offsetIndex.file().toPath()) + Files.size(timeIndex.file().toPath()) + Files.size(txnIndex.file().toPath());
+ } catch (IOException e) {
+ log.warn("Error occurred when estimating remote index cache entry bytes size, just set 0 firstly.", e);
+ return 0L;
+ }
+ }
Review Comment:
I think we should add `readLock` here to avoid getting incorrect size.
##########
storage/src/main/java/org/apache/kafka/storage/internals/log/RemoteIndexCache.java:
##########
@@ -669,4 +705,8 @@ public static String remoteTransactionIndexFileName(RemoteLogSegmentMetadata rem
return generateFileNamePrefixForIndex(remoteLogSegmentMetadata) + LogFileUtils.TXN_INDEX_FILE_SUFFIX;
}
+ public static int estimatedEntrySize(Entry entry) {
+ return entry.offsetIndex.sizeInBytes() + entry.timeIndex.sizeInBytes() + (int) entry.txnIndex.file().length();
Review Comment:
Please change to:
`return offsetIndex.sizeInBytes() + timeIndex.sizeInBytes() + Files.size(txnIndex.file().toPath());`
as @divijvaidya 's suggestion. The reason is when using `sizeInBytes`, we don't need read disk.
##########
core/src/test/scala/unit/kafka/log/remote/RemoteIndexCacheTest.scala:
##########
@@ -171,21 +178,24 @@ class RemoteIndexCacheTest {
// Calling getIndex on the same entry should not call rsm#fetchIndex again, but it should retrieve from cache
cache.getIndexEntry(metadataList.head)
assertCacheSize(1)
- verifyFetchIndexInvocation(count = 1)
+ // contains the 1 time for rsm when estimating entry size
+ verifyFetchIndexInvocation(count = 2)
Review Comment:
The comment in L178 is not correct now.
##########
storage/src/main/java/org/apache/kafka/storage/internals/log/RemoteIndexCache.java:
##########
@@ -125,12 +126,39 @@ public RemoteIndexCache(RemoteStorageManager remoteStorageManager, String logDir
* @param remoteStorageManager RemoteStorageManager instance, to be used in fetching indexes.
* @param logDir log directory
*/
- public RemoteIndexCache(int maxSize, RemoteStorageManager remoteStorageManager, String logDir) throws IOException {
+ public RemoteIndexCache(long maxSize, RemoteStorageManager remoteStorageManager, String logDir) throws IOException {
Review Comment:
Please update the javadoc for `maxSize`
##########
core/src/test/scala/unit/kafka/log/remote/RemoteIndexCacheTest.scala:
##########
@@ -171,21 +178,24 @@ class RemoteIndexCacheTest {
// Calling getIndex on the same entry should not call rsm#fetchIndex again, but it should retrieve from cache
cache.getIndexEntry(metadataList.head)
assertCacheSize(1)
- verifyFetchIndexInvocation(count = 1)
+ // contains the 1 time for rsm when estimating entry size
+ verifyFetchIndexInvocation(count = 2)
Review Comment:
Also, I don't understand why we still need to `fetchIndex` again here? If we still invoke fetchIndex under this situation, then the cache doesn't work as expected.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org