You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by at...@apache.org on 2023/06/06 05:41:59 UTC
[pinot] branch master updated: Support Off Heap for Native Text Indices (#10842)
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new f8766129f0 Support Off Heap for Native Text Indices (#10842)
f8766129f0 is described below
commit f8766129f06f29894b7e7c2dc1fed3b36af75d16
Author: Atri Sharma <at...@gmail.com>
AuthorDate: Tue Jun 6 11:11:53 2023 +0530
Support Off Heap for Native Text Indices (#10842)
This commit allows native text indices to be loaded off heap.
---
.../local/segment/index/readers/text/NativeTextIndexReader.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/NativeTextIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/NativeTextIndexReader.java
index 8f1c2e6a73..3650e3531f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/NativeTextIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/NativeTextIndexReader.java
@@ -52,7 +52,11 @@ public class NativeTextIndexReader implements TextIndexReader {
public NativeTextIndexReader(String column, File indexDir) {
_column = column;
try {
- _buffer = PinotDataBuffer.loadBigEndianFile(getTextIndexFile(indexDir));
+ String desc = "Native text index buffer: " + column;
+ File indexFile = getTextIndexFile(indexDir);
+ //TODO: Pass the load mode in (Direct, MMap)
+ _buffer =
+ PinotDataBuffer.mapFile(indexFile, /* readOnly */ true, 0, indexFile.length(), ByteOrder.BIG_ENDIAN, desc);
populateIndexes();
} catch (Exception e) {
LOGGER.error("Failed to instantiate Lucene text index reader for column {}, exception {}", column,
@@ -115,5 +119,6 @@ public class NativeTextIndexReader implements TextIndexReader {
@Override
public void close()
throws IOException {
+ _buffer.close();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org