You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2024/03/28 09:45:58 UTC
(lucene) branch branch_9x updated (e177b28fccc -> 72439d29cc0)
This is an automated email from the ASF dual-hosted git repository.
uschindler pushed a change to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
from e177b28fccc Speed up writeGroupVInts (#13203)
new ff166ac5311 Subtract deleted file size from the cache size of NRTCachingDirectory. (#13206)
new 72439d29cc0 fix typo in older changes entry
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
lucene/CHANGES.txt | 4 +++-
.../apache/lucene/store/NRTCachingDirectory.java | 13 ++++++++++++
.../lucene/store/TestNRTCachingDirectory.java | 24 ++++++++++++++++++++++
3 files changed, 40 insertions(+), 1 deletion(-)
(lucene) 01/02: Subtract deleted file size from the cache size of NRTCachingDirectory. (#13206)
Posted by us...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
uschindler pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit ff166ac5311fa3c356ad39b0d0203c1106930a32
Author: Jean-François BOEUF <je...@gmail.com>
AuthorDate: Thu Mar 28 10:34:46 2024 +0100
Subtract deleted file size from the cache size of NRTCachingDirectory. (#13206)
---
lucene/CHANGES.txt | 2 ++
.../apache/lucene/store/NRTCachingDirectory.java | 13 ++++++++++++
.../lucene/store/TestNRTCachingDirectory.java | 24 ++++++++++++++++++++++
3 files changed, 39 insertions(+)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 8e65a1075eb..044e6d0a72c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -78,6 +78,8 @@ Bug Fixes
* GITHUB#13204: Fix euals/hashCode of IOContext. (Uwe Schindler, Robert Muir)
+* GITHUB#13206: Subtract deleted file size from the cache size of NRTCachingDirectory. (Jean-François Boeuf)
+
Build
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java b/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
index d99b78c3b4b..f544193c79e 100644
--- a/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
@@ -72,6 +72,12 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
new SingleInstanceLockFactory(),
ByteBuffersDataOutput::new,
(fileName, content) -> {
+ // Defensive check to handle the case the file has been deleted before this lambda
+ // is called when the IndexOutput is closed. Unsafe in the unlikely case the deletion
+ // happens concurrently on another thread.
+ if (isCachedFile(fileName) == false) {
+ return null;
+ }
cacheSize.addAndGet(content.size());
return ByteBuffersDirectory.OUTPUT_AS_MANY_BUFFERS_LUCENE.apply(fileName, content);
});
@@ -123,7 +129,10 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
System.out.println("nrtdir.deleteFile name=" + name);
}
if (cacheDirectory.fileExists(name)) {
+ long size = cacheDirectory.fileLength(name);
cacheDirectory.deleteFile(name);
+ long newSize = cacheSize.addAndGet(-size);
+ assert newSize >= 0;
} else {
in.deleteFile(name);
}
@@ -300,6 +309,10 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
}
}
+ private synchronized boolean isCachedFile(String fileName) {
+ return cacheDirectory.fileExists(fileName);
+ }
+
private void unCache(String fileName) throws IOException {
// Must sync here because other sync methods have
// if (cache.fileNameExists(name)) { ... } else { ... }:
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java
index b81c3aeed6b..1588baf9be9 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java
@@ -34,6 +34,7 @@ import org.apache.lucene.tests.store.BaseDirectoryTestCase;
import org.apache.lucene.tests.util.LineFileDocs;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.BytesRef;
+import org.junit.Assert;
public class TestNRTCachingDirectory extends BaseDirectoryTestCase {
@@ -167,4 +168,27 @@ public class TestNRTCachingDirectory extends BaseDirectoryTestCase {
dir.close();
}
+
+ public void testCacheSizeAfterDelete() throws IOException {
+ IOContext ioContext = new IOContext(new FlushInfo(3, 40));
+ String fn = "f1";
+ try (Directory dir = newDirectory();
+ NRTCachingDirectory nrt = new NRTCachingDirectory(dir, 1, 1); ) {
+ // deletes a closed file
+ try (IndexOutput out = nrt.createOutput(fn, ioContext)) {
+ for (int i = 0; i < 10; i++) out.writeInt(i);
+ }
+ Assert.assertEquals(40, nrt.ramBytesUsed());
+ nrt.deleteFile(fn);
+ Assert.assertEquals(0, nrt.ramBytesUsed());
+
+ // Deletes an unclosed file (write before and after deletion
+ try (IndexOutput out = nrt.createOutput(fn, ioContext)) {
+ for (int i = 0; i < 10; i++) out.writeInt(i);
+ nrt.deleteFile(fn);
+ for (int i = 0; i < 10; i++) out.writeInt(i);
+ }
+ Assert.assertEquals(0, nrt.ramBytesUsed());
+ }
+ }
}
(lucene) 02/02: fix typo in older changes entry
Posted by us...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
uschindler pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 72439d29cc05a5ff7ddea0beadab585df2c58aa5
Author: Uwe Schindler <us...@apache.org>
AuthorDate: Thu Mar 28 10:39:34 2024 +0100
fix typo in older changes entry
---
lucene/CHANGES.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 044e6d0a72c..8b018ef3470 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -76,7 +76,7 @@ Bug Fixes
* GITHUB#13169: Fix potential race condition in DocumentsWriter & DocumentsWriterDeleteQueue (Ben Trent)
-* GITHUB#13204: Fix euals/hashCode of IOContext. (Uwe Schindler, Robert Muir)
+* GITHUB#13204: Fix equals/hashCode of IOContext. (Uwe Schindler, Robert Muir)
* GITHUB#13206: Subtract deleted file size from the cache size of NRTCachingDirectory. (Jean-François Boeuf)