You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by su...@apache.org on 2018/12/22 00:22:28 UTC
[incubator-pinot] branch trackBytesRead created (now 5e1ac4b)
This is an automated email from the ASF dual-hosted git repository.
sunithabeeram pushed a change to branch trackBytesRead
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 5e1ac4b [PINOT-7370] Return number of bytes read from the reader interfaces/implementations
This branch includes the following new commits:
new 5e1ac4b [PINOT-7370] Return number of bytes read from the reader interfaces/implementations
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: [PINOT-7370] Return number of bytes read
from the reader interfaces/implementations
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sunithabeeram pushed a commit to branch trackBytesRead
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 5e1ac4ba02219f99a7550aa2dfc6a0bca2c2c779
Author: Sunitha Beeram <sb...@sbeeram-ld2.linkedin.biz>
AuthorDate: Fri Dec 21 16:22:13 2018 -0800
[PINOT-7370] Return number of bytes read from the reader interfaces/implementations
---
.../pinot/core/common/BaseBlockValSet.java | 12 +++---
.../linkedin/pinot/core/common/BlockValSet.java | 19 +++-------
.../core/operator/docvalsets/SingleValueSet.java | 44 +++++++++++++---------
.../core/segment/index/readers/BaseDictionary.java | 28 ++++++++++----
.../segment/index/readers/BytesDictionary.java | 8 +++-
.../core/segment/index/readers/Dictionary.java | 12 +++---
.../segment/index/readers/StringDictionary.java | 8 +++-
7 files changed, 77 insertions(+), 54 deletions(-)
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/common/BaseBlockValSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/common/BaseBlockValSet.java
index f442755..f7a42ed 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/common/BaseBlockValSet.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/common/BaseBlockValSet.java
@@ -39,32 +39,32 @@ public abstract class BaseBlockValSet implements BlockValSet {
}
@Override
- public void getIntValues(int[] inDocIds, int inStartPos, int inDocIdsSize, int[] outValues, int outStartPos) {
+ public long getIntValues(int[] inDocIds, int inStartPos, int inDocIdsSize, int[] outValues, int outStartPos) {
throw new UnsupportedOperationException();
}
@Override
- public void getLongValues(int[] inDocIds, int inStartPos, int inDocIdsSize, long[] outValues, int outStartPos) {
+ public long getLongValues(int[] inDocIds, int inStartPos, int inDocIdsSize, long[] outValues, int outStartPos) {
throw new UnsupportedOperationException();
}
@Override
- public void getFloatValues(int[] inDocIds, int inStartPos, int inDocIdsSize, float[] outValues, int outStartPos) {
+ public long getFloatValues(int[] inDocIds, int inStartPos, int inDocIdsSize, float[] outValues, int outStartPos) {
throw new UnsupportedOperationException();
}
@Override
- public void getDoubleValues(int[] inDocIds, int inStartPos, int inDocIdsSize, double[] outValues, int outStartPos) {
+ public long getDoubleValues(int[] inDocIds, int inStartPos, int inDocIdsSize, double[] outValues, int outStartPos) {
throw new UnsupportedOperationException();
}
@Override
- public void getStringValues(int[] inDocIds, int inStartPos, int inDocIdsSize, String[] outValues, int outStartPos) {
+ public long getStringValues(int[] inDocIds, int inStartPos, int inDocIdsSize, String[] outValues, int outStartPos) {
throw new UnsupportedOperationException();
}
@Override
- public void getBytesValues(int[] inDocIds, int inStartPos, int inDocIdsSize, byte[][] outValues, int outStartPos) {
+ public long getBytesValues(int[] inDocIds, int inStartPos, int inDocIdsSize, byte[][] outValues, int outStartPos) {
throw new UnsupportedOperationException();
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/common/BlockValSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/common/BlockValSet.java
index 89aa7b2..6c6c8a7 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/common/BlockValSet.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/common/BlockValSet.java
@@ -30,7 +30,6 @@ public interface BlockValSet {
/**
* Get dictionary Ids for the given docIds.
- *
* @param inDocIds Input docIds
* @param inStartPos Start index in inDocIds
* @param inDocIdsSize Number of input doc ids
@@ -41,68 +40,62 @@ public interface BlockValSet {
/**
* Get Integer values for the given docIds.
- *
* @param inDocIds Input docIds
* @param inStartPos Start index in inDocIds
* @param inDocIdsSize Number of input doc ids
* @param outValues Output array
* @param outStartPos Start position in outValues
*/
- void getIntValues(int[] inDocIds, int inStartPos, int inDocIdsSize, int[] outValues, int outStartPos);
+ long getIntValues(int[] inDocIds, int inStartPos, int inDocIdsSize, int[] outValues, int outStartPos);
/**
* Get long values for the given docIds.
- *
* @param inDocIds Input docIds
* @param inStartPos Start index in inDocIds
* @param inDocIdsSize Number of input doc ids
* @param outValues Output array
* @param outStartPos Start position in outValues
*/
- void getLongValues(int[] inDocIds, int inStartPos, int inDocIdsSize, long[] outValues, int outStartPos);
+ long getLongValues(int[] inDocIds, int inStartPos, int inDocIdsSize, long[] outValues, int outStartPos);
/**
* Get float values for the given docIds.
- *
* @param inDocIds Input docIds
* @param inStartPos Start index in inDocIds
* @param inDocIdsSize Number of input doc ids
* @param outValues Output array
* @param outStartPos Start position in outValues
*/
- void getFloatValues(int[] inDocIds, int inStartPos, int inDocIdsSize, float[] outValues, int outStartPos);
+ long getFloatValues(int[] inDocIds, int inStartPos, int inDocIdsSize, float[] outValues, int outStartPos);
/**
- *
* @param inDocIds Input docIds
* @param inStartPos Start index in inDocIds
* @param inDocIdsSize Number of input doc ids
* @param outValues Output array
* @param outStartPos Start position in outValues
*/
- void getDoubleValues(int[] inDocIds, int inStartPos, int inDocIdsSize, double[] outValues, int outStartPos);
+ long getDoubleValues(int[] inDocIds, int inStartPos, int inDocIdsSize, double[] outValues, int outStartPos);
/**
* Get string values for the given docIds.
- *
* @param inDocIds Input docIds
* @param inStartPos Start index in inDocIds
* @param inDocIdsSize Number of input doc ids
* @param outValues Output array
* @param outStartPos Start position in outValues
*/
- void getStringValues(int[] inDocIds, int inStartPos, int inDocIdsSize, String[] outValues, int outStartPos);
+ long getStringValues(int[] inDocIds, int inStartPos, int inDocIdsSize, String[] outValues, int outStartPos);
/**
* Get byte[] values for the given docIds.
- *
* @param inDocIds Input docIds
* @param inStartPos Start index in inDocIds
* @param inDocIdsSize Number of input doc ids
* @param outValues Output array
* @param outStartPos Start position in outValues
*/
- void getBytesValues(int[] inDocIds, int inStartPos, int inDocIdsSize, byte[][] outValues, int outStartPos);
+ long getBytesValues(int[] inDocIds, int inStartPos, int inDocIdsSize, byte[][] outValues, int outStartPos);
/**
* SINGLE-VALUED COLUMN APIs
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docvalsets/SingleValueSet.java b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docvalsets/SingleValueSet.java
index 0480454..0fd13d6 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docvalsets/SingleValueSet.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/operator/docvalsets/SingleValueSet.java
@@ -46,7 +46,7 @@ public final class SingleValueSet extends BaseBlockValSet {
}
@Override
- public void getIntValues(int[] inDocIds, int inStartPos, int inDocIdsSize, int[] outValues, int outStartPos) {
+ public long getIntValues(int[] inDocIds, int inStartPos, int inDocIdsSize, int[] outValues, int outStartPos) {
int inEndPos = inStartPos + inDocIdsSize;
ReaderContext context = _reader.createContext();
if (_dataType == DataType.INT) {
@@ -56,10 +56,11 @@ public final class SingleValueSet extends BaseBlockValSet {
} else {
throw new UnsupportedOperationException();
}
+ return inDocIdsSize * Integer.BYTES;
}
@Override
- public void getLongValues(int[] inDocIds, int inStartPos, int inDocIdsSize, long[] outValues, int outStartPos) {
+ public long getLongValues(int[] inDocIds, int inStartPos, int inDocIdsSize, long[] outValues, int outStartPos) {
int inEndPos = inStartPos + inDocIdsSize;
ReaderContext context = _reader.createContext();
switch (_dataType) {
@@ -67,19 +68,19 @@ public final class SingleValueSet extends BaseBlockValSet {
for (int i = inStartPos; i < inEndPos; i++) {
outValues[outStartPos++] = _reader.getInt(inDocIds[i], context);
}
- break;
+ return inDocIdsSize * Integer.BYTES;
case LONG:
for (int i = inStartPos; i < inEndPos; i++) {
outValues[outStartPos++] = _reader.getLong(inDocIds[i], context);
}
- break;
+ return inDocIdsSize * Long.BYTES;
default:
throw new UnsupportedOperationException();
}
}
@Override
- public void getFloatValues(int[] inDocIds, int inStartPos, int inDocIdsSize, float[] outValues, int outStartPos) {
+ public long getFloatValues(int[] inDocIds, int inStartPos, int inDocIdsSize, float[] outValues, int outStartPos) {
int inEndPos = inStartPos + inDocIdsSize;
ReaderContext context = _reader.createContext();
switch (_dataType) {
@@ -87,24 +88,24 @@ public final class SingleValueSet extends BaseBlockValSet {
for (int i = inStartPos; i < inEndPos; i++) {
outValues[outStartPos++] = _reader.getInt(inDocIds[i], context);
}
- break;
+ return inDocIdsSize * Integer.BYTES;
case LONG:
for (int i = inStartPos; i < inEndPos; i++) {
outValues[outStartPos++] = _reader.getLong(inDocIds[i], context);
}
- break;
+ return inDocIdsSize * Long.BYTES;
case FLOAT:
for (int i = inStartPos; i < inEndPos; i++) {
outValues[outStartPos++] = _reader.getFloat(inDocIds[i], context);
}
- break;
+ return inDocIdsSize * Float.BYTES;
default:
throw new UnsupportedOperationException();
}
}
@Override
- public void getDoubleValues(int[] inDocIds, int inStartPos, int inDocIdsSize, double[] outValues, int outStartPos) {
+ public long getDoubleValues(int[] inDocIds, int inStartPos, int inDocIdsSize, double[] outValues, int outStartPos) {
int inEndPos = inStartPos + inDocIdsSize;
ReaderContext context = _reader.createContext();
switch (_dataType) {
@@ -112,48 +113,57 @@ public final class SingleValueSet extends BaseBlockValSet {
for (int i = inStartPos; i < inEndPos; i++) {
outValues[outStartPos++] = _reader.getInt(inDocIds[i], context);
}
- break;
+ return inDocIdsSize * Integer.BYTES;
case LONG:
for (int i = inStartPos; i < inEndPos; i++) {
outValues[outStartPos++] = _reader.getLong(inDocIds[i], context);
}
- break;
+ return inDocIdsSize * Long.BYTES;
case FLOAT:
for (int i = inStartPos; i < inEndPos; i++) {
outValues[outStartPos++] = _reader.getFloat(inDocIds[i], context);
}
- break;
+ return inDocIdsSize * Float.BYTES;
case DOUBLE:
for (int i = inStartPos; i < inEndPos; i++) {
outValues[outStartPos++] = _reader.getDouble(inDocIds[i], context);
}
- break;
+ return inDocIdsSize * Double.BYTES;
default:
throw new UnsupportedOperationException();
}
}
@Override
- public void getStringValues(int[] inDocIds, int inStartPos, int inDocIdsSize, String[] outValues, int outStartPos) {
+ public long getStringValues(int[] inDocIds, int inStartPos, int inDocIdsSize, String[] outValues, int outStartPos) {
+
+ long bytesRead = 0;
int inEndPos = inStartPos + inDocIdsSize;
ReaderContext context = _reader.createContext();
if (_dataType == DataType.STRING) {
for (int i = inStartPos; i < inEndPos; i++) {
- outValues[outStartPos++] = _reader.getString(inDocIds[i], context);
+ String val = _reader.getString(inDocIds[i], context);
+ outValues[outStartPos++] = val;
+ bytesRead += val.length();
}
+ return bytesRead;
} else {
throw new UnsupportedOperationException();
}
}
@Override
- public void getBytesValues(int[] inDocIds, int inStartPos, int inDocIdsSize, byte[][] outValues, int outStartPos) {
+ public long getBytesValues(int[] inDocIds, int inStartPos, int inDocIdsSize, byte[][] outValues, int outStartPos) {
int inEndPos = inStartPos + inDocIdsSize;
+ long bytesRead = 0;
ReaderContext context = _reader.createContext();
if (_dataType.equals(DataType.BYTES)) {
for (int i = inStartPos; i < inEndPos; i++) {
- outValues[outStartPos++] = _reader.getBytes(inDocIds[i], context);
+ byte[] val = _reader.getBytes(inDocIds[i], context);
+ outValues[outStartPos++] = val;
+ bytesRead += val.length;
}
+ return bytesRead;
} else {
throw new UnsupportedOperationException();
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BaseDictionary.java b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BaseDictionary.java
index 181168c..b919884 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BaseDictionary.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BaseDictionary.java
@@ -54,50 +54,62 @@ public abstract class BaseDictionary implements Dictionary {
}
@Override
- public void readIntValues(int[] dictIds, int inStartPos, int length, int[] outValues, int outStartPos) {
+ public long readIntValues(int[] dictIds, int inStartPos, int length, int[] outValues, int outStartPos) {
int inEndPos = inStartPos + length;
for (int i = inStartPos; i < inEndPos; i++) {
outValues[outStartPos++] = getIntValue(dictIds[i]);
}
+ return length * Integer.BYTES;
}
@Override
- public void readLongValues(int[] dictIds, int inStartPos, int length, long[] outValues, int outStartPos) {
+ public long readLongValues(int[] dictIds, int inStartPos, int length, long[] outValues, int outStartPos) {
int inEndPos = inStartPos + length;
for (int i = inStartPos; i < inEndPos; i++) {
outValues[outStartPos++] = getLongValue(dictIds[i]);
}
+ return length * Long.BYTES;
}
@Override
- public void readFloatValues(int[] dictIds, int inStartPos, int length, float[] outValues, int outStartPos) {
+ public long readFloatValues(int[] dictIds, int inStartPos, int length, float[] outValues, int outStartPos) {
int inEndPos = inStartPos + length;
for (int i = inStartPos; i < inEndPos; i++) {
outValues[outStartPos++] = getFloatValue(dictIds[i]);
}
+ return length * Float.BYTES;
}
@Override
- public void readDoubleValues(int[] dictIds, int inStartPos, int length, double[] outValues, int outStartPos) {
+ public long readDoubleValues(int[] dictIds, int inStartPos, int length, double[] outValues, int outStartPos) {
int inEndPos = inStartPos + length;
for (int i = inStartPos; i < inEndPos; i++) {
outValues[outStartPos++] = getDoubleValue(dictIds[i]);
}
+ return length * Double.BYTES;
}
@Override
- public void readStringValues(int[] dictIds, int inStartPos, int length, String[] outValues, int outStartPos) {
+ public long readStringValues(int[] dictIds, int inStartPos, int length, String[] outValues, int outStartPos) {
+ long bytesRead = 0;
int inEndPos = inStartPos + length;
for (int i = inStartPos; i < inEndPos; i++) {
- outValues[outStartPos++] = getStringValue(dictIds[i]);
+ String str = getStringValue(dictIds[i]);
+ outValues[outStartPos++] = str;
+ bytesRead += str.length();
}
+ return bytesRead;
}
@Override
- public void readBytesValues(int[] dictIds, int inStartPos, int length, byte[][] outValues, int outStartPos) {
+ public long readBytesValues(int[] dictIds, int inStartPos, int length, byte[][] outValues, int outStartPos) {
+ long bytesRead = 0;
int inEndPos = inStartPos + length;
for (int i = inStartPos; i < inEndPos; i++) {
- outValues[outStartPos++] = getBytesValue(dictIds[i]);
+ byte[] val = getBytesValue(dictIds[i]);
+ outValues[outStartPos++] = val;
+ bytesRead += val.length;
}
+ return bytesRead;
}
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BytesDictionary.java b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BytesDictionary.java
index b5cc02c..bce06cd 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BytesDictionary.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/BytesDictionary.java
@@ -49,10 +49,14 @@ public class BytesDictionary extends ImmutableDictionaryReader {
}
@Override
- public void readBytesValues(int[] dictIds, int inStartPos, int length, byte[][] outValues, int outStartPos) {
+ public long readBytesValues(int[] dictIds, int inStartPos, int length, byte[][] outValues, int outStartPos) {
+ long bytesRead = 0;
int inEndPos = inStartPos + length;
for (int i = inStartPos; i < inEndPos; i++) {
- outValues[outStartPos++] = getBytes(dictIds[i], getBuffer());
+ byte[] val = getBytes(dictIds[i], getBuffer());
+ outValues[outStartPos++] = val;
+ bytesRead += val.length;
}
+ return bytesRead;
}
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/Dictionary.java b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/Dictionary.java
index 80145fb..257749f 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/Dictionary.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/Dictionary.java
@@ -50,16 +50,16 @@ public interface Dictionary extends Closeable {
// Batch read APIs
- void readIntValues(int[] dictIds, int inStartPos, int length, int[] outValues, int outStartPos);
+ long readIntValues(int[] dictIds, int inStartPos, int length, int[] outValues, int outStartPos);
- void readLongValues(int[] dictIds, int inStartPos, int length, long[] outValues, int outStartPos);
+ long readLongValues(int[] dictIds, int inStartPos, int length, long[] outValues, int outStartPos);
- void readFloatValues(int[] dictIds, int inStartPos, int length, float[] outValues, int outStartPos);
+ long readFloatValues(int[] dictIds, int inStartPos, int length, float[] outValues, int outStartPos);
- void readDoubleValues(int[] dictIds, int inStartPos, int length, double[] outValues, int outStartPos);
+ long readDoubleValues(int[] dictIds, int inStartPos, int length, double[] outValues, int outStartPos);
- void readStringValues(int[] dictIds, int inStartPos, int length, String[] outValues, int outStartPos);
+ long readStringValues(int[] dictIds, int inStartPos, int length, String[] outValues, int outStartPos);
- void readBytesValues(int[] dictIds, int inStartPos, int length, byte[][] outValues, int outStartPos);
+ long readBytesValues(int[] dictIds, int inStartPos, int length, byte[][] outValues, int outStartPos);
}
diff --git a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/StringDictionary.java b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/StringDictionary.java
index 2460547..7a7dab1 100644
--- a/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/StringDictionary.java
+++ b/pinot-core/src/main/java/com/linkedin/pinot/core/segment/index/readers/StringDictionary.java
@@ -51,11 +51,15 @@ public class StringDictionary extends ImmutableDictionaryReader {
}
@Override
- public void readStringValues(int[] dictIds, int inStartPos, int length, String[] outValues, int outStartPos) {
+ public long readStringValues(int[] dictIds, int inStartPos, int length, String[] outValues, int outStartPos) {
+ long bytesRead = 0;
byte[] buffer = getBuffer();
int inEndPos = inStartPos + length;
for (int i = inStartPos; i < inEndPos; i++) {
- outValues[outStartPos++] = getUnpaddedString(dictIds[i], buffer);
+ String val = getUnpaddedString(dictIds[i], buffer);
+ outValues[outStartPos++] = val;
+ bytesRead += val.length();
}
+ return bytesRead;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org