You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by sn...@apache.org on 2021/01/26 00:33:58 UTC
[incubator-pinot] 01/01: Fix the overflow issue when loading the
large dictionary into the buffer (#6476)
This is an automated email from the ASF dual-hosted git repository.
snlee pushed a commit to branch hotfix-overflow-2gb
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 991e4b1c69c43a9ab968c431e7f5a8c34317fca6
Author: Seunghyun Lee <sn...@linkedin.com>
AuthorDate: Mon Jan 25 12:37:51 2021 -0800
Fix the overflow issue when loading the large dictionary into the buffer (#6476)
Currently, we allow to generate the large dictionary (>2GB) while loading
segment fails due to overflow. This pr fixes the issue.
---
.../apache/pinot/core/segment/store/SingleFileIndexDirectory.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/store/SingleFileIndexDirectory.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/store/SingleFileIndexDirectory.java
index 6ba81f8..c34ad2f 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/segment/store/SingleFileIndexDirectory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/store/SingleFileIndexDirectory.java
@@ -180,7 +180,7 @@ class SingleFileIndexDirectory extends ColumnIndexDirectory {
}
}
- private void validateMagicMarker(PinotDataBuffer buffer, int startOffset) {
+ private void validateMagicMarker(PinotDataBuffer buffer, long startOffset) {
long actualMarkerValue = buffer.getLong(startOffset);
if (actualMarkerValue != MAGIC_MARKER) {
LOGGER.error("Missing magic marker in index file: {} at position: {}", indexFile, startOffset);
@@ -292,10 +292,10 @@ class SingleFileIndexDirectory extends ColumnIndexDirectory {
}
allocBuffers.add(buffer);
- int prevSlicePoint = 0;
+ long prevSlicePoint = 0;
for (Long fileOffset : offsetAccum) {
IndexEntry entry = startOffsets.get(fileOffset);
- int endSlicePoint = prevSlicePoint + (int) entry.size;
+ long endSlicePoint = prevSlicePoint + entry.size;
validateMagicMarker(buffer, prevSlicePoint);
PinotDataBuffer viewBuffer = buffer.view(prevSlicePoint + MAGIC_MARKER_SIZE_BYTES, endSlicePoint);
entry.buffer = viewBuffer;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org