You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by mc...@apache.org on 2018/11/19 16:58:51 UTC
[incubator-pinot] branch master updated: Let
PinotDataBuffer.newIndexFor(...) takes long value (#3512)
This is an automated email from the ASF dual-hosted git repository.
mcvsubbu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new ac915e4 Let PinotDataBuffer.newIndexFor(...) takes long value (#3512)
ac915e4 is described below
commit ac915e451066ae5f5c4fc19f02c5cca7b148ed35
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Mon Nov 19 08:58:46 2018 -0800
Let PinotDataBuffer.newIndexFor(...) takes long value (#3512)
* Let PinotDataBuffer.newIndexFor(...) takes long value
* Make PinotDataBuffer takes long value
---
.../index/converter/SegmentV1V2ToV3FormatConverter.java | 2 +-
.../linkedin/pinot/core/segment/index/loader/LoaderUtils.java | 2 +-
.../pinot/core/segment/store/ColumnIndexDirectory.java | 6 +++---
.../pinot/core/segment/store/FilePerIndexDirectory.java | 10 +++++-----
.../linkedin/pinot/core/segment/store/SegmentDirectory.java | 2 +-
.../pinot/core/segment/store/SegmentLocalFSDirectory.java | 9 ++++-----
.../pinot/core/segment/store/SingleFileIndexDirectory.java | 8 ++++----
7 files changed, 19 insertions(+), 20 deletions(-)
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/converter/SegmentV1V2ToV3FormatConverter.java b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
index 833059c..da1b23c 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/converter/SegmentV1V2ToV3FormatConverter.java
@@ -198,7 +198,7 @@ public class SegmentV1V2ToV3FormatConverter implements SegmentFormatConverter {
private void readCopyBuffers(SegmentDirectory.Reader reader, SegmentDirectory.Writer writer, String column,
ColumnIndexType indexType) throws IOException {
PinotDataBuffer oldBuffer = reader.getIndexFor(column, indexType);
- PinotDataBuffer newDictBuffer = writer.newIndexFor(column, indexType, (int) oldBuffer.size());
+ PinotDataBuffer newDictBuffer = writer.newIndexFor(column, indexType, oldBuffer.size());
oldBuffer.copyTo(0, newDictBuffer, 0, oldBuffer.size());
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/loader/LoaderUtils.java b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/loader/LoaderUtils.java
index 2ba1a4d..b8c30e4 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/loader/LoaderUtils.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/loader/LoaderUtils.java
@@ -48,7 +48,7 @@ public class LoaderUtils {
*/
public static void writeIndexToV3Format(SegmentDirectory.Writer segmentWriter, String column, File indexFile,
ColumnIndexType indexType) throws IOException {
- int fileLength = (int) indexFile.length();
+ long fileLength = indexFile.length();
try (PinotDataBuffer buffer = segmentWriter.newIndexFor(column, indexType, fileLength)) {
buffer.readFrom(0, indexFile, 0, fileLength);
FileUtils.forceDelete(indexFile);
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/ColumnIndexDirectory.java b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/ColumnIndexDirectory.java
index c20a091..3579f1c 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/ColumnIndexDirectory.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/ColumnIndexDirectory.java
@@ -101,7 +101,7 @@ abstract class ColumnIndexDirectory implements Closeable {
* @return in-memory ByteBuffer like buffer for data
* @throws IOException
*/
- public abstract PinotDataBuffer newDictionaryBuffer(String column, int sizeBytes)
+ public abstract PinotDataBuffer newDictionaryBuffer(String column, long sizeBytes)
throws IOException;
/**
* Allocate a new data buffer of specified sizeBytes in the columnar index directory
@@ -110,7 +110,7 @@ abstract class ColumnIndexDirectory implements Closeable {
* @return in-memory ByteBuffer like buffer for data
* @throws IOException
*/
- public abstract PinotDataBuffer newForwardIndexBuffer(String column, int sizeBytes)
+ public abstract PinotDataBuffer newForwardIndexBuffer(String column, long sizeBytes)
throws IOException;
/**
* Allocate a new data buffer of specified sizeBytes in the columnar index directory
@@ -119,7 +119,7 @@ abstract class ColumnIndexDirectory implements Closeable {
* @return in-memory ByteBuffer like buffer for data
* @throws IOException
*/
- public abstract PinotDataBuffer newInvertedIndexBuffer(String column, int sizeBytes)
+ public abstract PinotDataBuffer newInvertedIndexBuffer(String column, long sizeBytes)
throws IOException;
/**
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/FilePerIndexDirectory.java b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/FilePerIndexDirectory.java
index 6c99582..c6ea2d9 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/FilePerIndexDirectory.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/FilePerIndexDirectory.java
@@ -46,7 +46,7 @@ class FilePerIndexDirectory extends ColumnIndexDirectory {
}
@Override
- public PinotDataBuffer newDictionaryBuffer(String column, int sizeBytes)
+ public PinotDataBuffer newDictionaryBuffer(String column, long sizeBytes)
throws IOException {
IndexKey key = new IndexKey(column, ColumnIndexType.DICTIONARY);
return getWriteBufferFor(key, sizeBytes);
@@ -60,7 +60,7 @@ class FilePerIndexDirectory extends ColumnIndexDirectory {
}
@Override
- public PinotDataBuffer newForwardIndexBuffer(String column, int sizeBytes)
+ public PinotDataBuffer newForwardIndexBuffer(String column, long sizeBytes)
throws IOException {
IndexKey key = new IndexKey(column, ColumnIndexType.FORWARD_INDEX);
return getWriteBufferFor(key, sizeBytes);
@@ -74,7 +74,7 @@ class FilePerIndexDirectory extends ColumnIndexDirectory {
}
@Override
- public PinotDataBuffer newInvertedIndexBuffer(String column, int sizeBytes)
+ public PinotDataBuffer newInvertedIndexBuffer(String column, long sizeBytes)
throws IOException {
IndexKey key = new IndexKey(column, ColumnIndexType.INVERTED_INDEX);
return getWriteBufferFor(key, sizeBytes);
@@ -116,7 +116,7 @@ class FilePerIndexDirectory extends ColumnIndexDirectory {
return buffer;
}
- private PinotDataBuffer getWriteBufferFor(IndexKey key, int sizeBytes) throws IOException {
+ private PinotDataBuffer getWriteBufferFor(IndexKey key, long sizeBytes) throws IOException {
if (indexBuffers.containsKey(key)) {
return indexBuffers.get(key);
}
@@ -146,7 +146,7 @@ class FilePerIndexDirectory extends ColumnIndexDirectory {
return new File(segmentDirectory, filename);
}
- private PinotDataBuffer mapForWrites(File file, int sizeBytes, String context) throws IOException {
+ private PinotDataBuffer mapForWrites(File file, long sizeBytes, String context) throws IOException {
Preconditions.checkNotNull(file);
Preconditions.checkArgument(sizeBytes >= 0 && sizeBytes < Integer.MAX_VALUE,
"File size must be less than 2GB, file: " + file);
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentDirectory.java b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentDirectory.java
index 8881ea5..918e61f 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentDirectory.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentDirectory.java
@@ -181,7 +181,7 @@ public abstract class SegmentDirectory implements Closeable {
// NOTE: an interface like readFrom(File f, String column, ColumnIndexType, int sizeBytes) will be safe
// but it can lead to potential endianness issues. Endianness used to create data may not be
// same as PinotDataBufferOld
- public abstract PinotDataBuffer newIndexFor(String columnName, ColumnIndexType indexType, int sizeBytes)
+ public abstract PinotDataBuffer newIndexFor(String columnName, ColumnIndexType indexType, long sizeBytes)
throws IOException;
/**
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentLocalFSDirectory.java b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentLocalFSDirectory.java
index 37cb57f..d29c37c 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentLocalFSDirectory.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SegmentLocalFSDirectory.java
@@ -356,7 +356,7 @@ class SegmentLocalFSDirectory extends SegmentDirectory {
}
@Override
- public PinotDataBuffer newIndexFor(String columnName, ColumnIndexType indexType, int sizeBytes)
+ public PinotDataBuffer newIndexFor(String columnName, ColumnIndexType indexType, long sizeBytes)
throws IOException {
return getNewIndexBuffer(new IndexKey(columnName, indexType), sizeBytes);
}
@@ -407,12 +407,11 @@ class SegmentLocalFSDirectory extends SegmentDirectory {
ColumnIndexType indexType = key.type;
switch (indexType) {
case DICTIONARY:
- return columnIndexDirectory.newDictionaryBuffer(key.name, (int) sizeBytes);
-
+ return columnIndexDirectory.newDictionaryBuffer(key.name, sizeBytes);
case FORWARD_INDEX:
- return columnIndexDirectory.newForwardIndexBuffer(key.name, (int) sizeBytes);
+ return columnIndexDirectory.newForwardIndexBuffer(key.name, sizeBytes);
case INVERTED_INDEX:
- return columnIndexDirectory.newInvertedIndexBuffer(key.name, ((int) sizeBytes));
+ return columnIndexDirectory.newInvertedIndexBuffer(key.name, sizeBytes);
default:
throw new RuntimeException("Unknown index type: " + indexType.name() +
" for directory: " + segmentDirectory);
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SingleFileIndexDirectory.java b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SingleFileIndexDirectory.java
index 127f437..cf22cc6 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SingleFileIndexDirectory.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/store/SingleFileIndexDirectory.java
@@ -111,19 +111,19 @@ class SingleFileIndexDirectory extends ColumnIndexDirectory {
}
@Override
- public PinotDataBuffer newDictionaryBuffer(String column, int sizeBytes)
+ public PinotDataBuffer newDictionaryBuffer(String column, long sizeBytes)
throws IOException {
return allocNewBufferInternal(column, ColumnIndexType.DICTIONARY, sizeBytes, "dictionary.create");
}
@Override
- public PinotDataBuffer newForwardIndexBuffer(String column, int sizeBytes)
+ public PinotDataBuffer newForwardIndexBuffer(String column, long sizeBytes)
throws IOException {
return allocNewBufferInternal(column, ColumnIndexType.FORWARD_INDEX, sizeBytes, "forward_index.create");
}
@Override
- public PinotDataBuffer newInvertedIndexBuffer(String column, int sizeBytes)
+ public PinotDataBuffer newInvertedIndexBuffer(String column, long sizeBytes)
throws IOException {
return allocNewBufferInternal(column, ColumnIndexType.INVERTED_INDEX, sizeBytes, "inverted_index.create");
}
@@ -139,7 +139,7 @@ class SingleFileIndexDirectory extends ColumnIndexDirectory {
}
// This is using extra resources right now which can be changed.
- private PinotDataBuffer allocNewBufferInternal(String column, ColumnIndexType indexType, int size,
+ private PinotDataBuffer allocNewBufferInternal(String column, ColumnIndexType indexType, long size,
String context)
throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org