You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ro...@apache.org on 2022/06/21 00:27:06 UTC
[pinot] branch master updated: reduce positioning operation to single operation (#8926)
This is an automated email from the ASF dual-hosted git repository.
rongr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 291ed91797 reduce positioning operation to single operation (#8926)
291ed91797 is described below
commit 291ed9179775bdc01a22ce5582268c76408e62bd
Author: Rong Rong <ro...@apache.org>
AuthorDate: Mon Jun 20 17:27:00 2022 -0700
reduce positioning operation to single operation (#8926)
* use direct positioning API
- replacing current way, where ByteBuffer is positioned then access at that position.
- not replacing variable ByteBuffer positioning yet.
- fixing positioning issue in variable byte buffer.
* address diff comments
Co-authored-by: Rong Rong <ro...@startree.ai>
---
.../pinot/core/common/datablock/BaseDataBlock.java | 40 ++++++++++------------
.../core/common/datablock/ColumnarDataBlock.java | 13 ++++---
.../pinot/core/common/datablock/MetadataBlock.java | 4 +--
.../pinot/core/common/datablock/RowDataBlock.java | 13 ++++---
.../pinot/core/common/datatable/BaseDataTable.java | 21 +++++-------
5 files changed, 40 insertions(+), 51 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/BaseDataBlock.java b/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/BaseDataBlock.java
index dff60efac9..aca905b832 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/BaseDataBlock.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/BaseDataBlock.java
@@ -207,21 +207,22 @@ public abstract class BaseDataBlock implements DataTable {
protected abstract int getDataBlockVersionType();
/**
- * position the {@code _fixedSizeDataBytes} member variable to the corresponding row/column ID.
+ * return the offset in {@code _fixedSizeDataBytes} of the row/column ID.
* @param rowId row ID
* @param colId column ID
+ * @return the offset in the fixed size buffer for the row/columnID.
*/
- protected abstract void positionCursorInFixSizedBuffer(int rowId, int colId);
+ protected abstract int getOffsetInFixedBuffer(int rowId, int colId);
/**
- * position the {@code _variableSizeDataBytes} member variable to the corresponding row/column ID. and return the
+ * position the {@code _variableSizeDataBytes} to the corresponding row/column ID. and return the
* length of bytes to extract from the variable size buffer.
*
* @param rowId row ID
* @param colId column ID
* @return the length to extract from variable size buffer.
*/
- protected abstract int positionCursorInVariableBuffer(int rowId, int colId);
+ protected abstract int positionOffsetInVariableBufferAndGetLength(int rowId, int colId);
@Override
public Map<String, String> getMetadata() {
@@ -249,31 +250,27 @@ public abstract class BaseDataBlock implements DataTable {
@Override
public int getInt(int rowId, int colId) {
- positionCursorInFixSizedBuffer(rowId, colId);
- return _fixedSizeData.getInt();
+ return _fixedSizeData.getInt(getOffsetInFixedBuffer(rowId, colId));
}
@Override
public long getLong(int rowId, int colId) {
- positionCursorInFixSizedBuffer(rowId, colId);
- return _fixedSizeData.getLong();
+ return _fixedSizeData.getLong(getOffsetInFixedBuffer(rowId, colId));
}
@Override
public float getFloat(int rowId, int colId) {
- positionCursorInFixSizedBuffer(rowId, colId);
- return _fixedSizeData.getFloat();
+ return _fixedSizeData.getFloat(getOffsetInFixedBuffer(rowId, colId));
}
@Override
public double getDouble(int rowId, int colId) {
- positionCursorInFixSizedBuffer(rowId, colId);
- return _fixedSizeData.getDouble();
+ return _fixedSizeData.getDouble(getOffsetInFixedBuffer(rowId, colId));
}
@Override
public BigDecimal getBigDecimal(int rowId, int colId) {
- int size = positionCursorInVariableBuffer(rowId, colId);
+ int size = positionOffsetInVariableBufferAndGetLength(rowId, colId);
ByteBuffer byteBuffer = _variableSizeData.slice();
byteBuffer.limit(size);
return BigDecimalUtils.deserialize(byteBuffer);
@@ -281,13 +278,12 @@ public abstract class BaseDataBlock implements DataTable {
@Override
public String getString(int rowId, int colId) {
- positionCursorInFixSizedBuffer(rowId, colId);
- return _stringDictionary[_fixedSizeData.getInt()];
+ return _stringDictionary[_fixedSizeData.getInt(getOffsetInFixedBuffer(rowId, colId))];
}
@Override
public ByteArray getBytes(int rowId, int colId) {
- int size = positionCursorInVariableBuffer(rowId, colId);
+ int size = positionOffsetInVariableBufferAndGetLength(rowId, colId);
byte[] buffer = new byte[size];
_variableSizeData.get(buffer);
return new ByteArray(buffer);
@@ -299,7 +295,7 @@ public abstract class BaseDataBlock implements DataTable {
@Override
public <T> T getObject(int rowId, int colId) {
- int size = positionCursorInVariableBuffer(rowId, colId);
+ int size = positionOffsetInVariableBufferAndGetLength(rowId, colId);
int objectTypeValue = _variableSizeData.getInt();
if (size == 0) {
assert objectTypeValue == ObjectSerDeUtils.ObjectType.Null.getValue();
@@ -312,7 +308,7 @@ public abstract class BaseDataBlock implements DataTable {
@Override
public int[] getIntArray(int rowId, int colId) {
- int length = positionCursorInVariableBuffer(rowId, colId);
+ int length = positionOffsetInVariableBufferAndGetLength(rowId, colId);
int[] ints = new int[length];
for (int i = 0; i < length; i++) {
ints[i] = _variableSizeData.getInt();
@@ -322,7 +318,7 @@ public abstract class BaseDataBlock implements DataTable {
@Override
public long[] getLongArray(int rowId, int colId) {
- int length = positionCursorInVariableBuffer(rowId, colId);
+ int length = positionOffsetInVariableBufferAndGetLength(rowId, colId);
long[] longs = new long[length];
for (int i = 0; i < length; i++) {
longs[i] = _variableSizeData.getLong();
@@ -332,7 +328,7 @@ public abstract class BaseDataBlock implements DataTable {
@Override
public float[] getFloatArray(int rowId, int colId) {
- int length = positionCursorInVariableBuffer(rowId, colId);
+ int length = positionOffsetInVariableBufferAndGetLength(rowId, colId);
float[] floats = new float[length];
for (int i = 0; i < length; i++) {
floats[i] = _variableSizeData.getFloat();
@@ -342,7 +338,7 @@ public abstract class BaseDataBlock implements DataTable {
@Override
public double[] getDoubleArray(int rowId, int colId) {
- int length = positionCursorInVariableBuffer(rowId, colId);
+ int length = positionOffsetInVariableBufferAndGetLength(rowId, colId);
double[] doubles = new double[length];
for (int i = 0; i < length; i++) {
doubles[i] = _variableSizeData.getDouble();
@@ -352,7 +348,7 @@ public abstract class BaseDataBlock implements DataTable {
@Override
public String[] getStringArray(int rowId, int colId) {
- int length = positionCursorInVariableBuffer(rowId, colId);
+ int length = positionOffsetInVariableBufferAndGetLength(rowId, colId);
String[] strings = new String[length];
for (int i = 0; i < length; i++) {
strings[i] = _stringDictionary[_variableSizeData.getInt()];
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/ColumnarDataBlock.java b/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/ColumnarDataBlock.java
index a83b8b2a30..9a3cff57f7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/ColumnarDataBlock.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/ColumnarDataBlock.java
@@ -66,16 +66,15 @@ public class ColumnarDataBlock extends BaseDataBlock {
}
@Override
- protected void positionCursorInFixSizedBuffer(int rowId, int colId) {
- int position = _cumulativeColumnOffsetSizeInBytes[colId] + _columnSizeInBytes[colId] * rowId;
- _fixedSizeData.position(position);
+ protected int getOffsetInFixedBuffer(int rowId, int colId) {
+ return _cumulativeColumnOffsetSizeInBytes[colId] + _columnSizeInBytes[colId] * rowId;
}
@Override
- protected int positionCursorInVariableBuffer(int rowId, int colId) {
- positionCursorInFixSizedBuffer(rowId, colId);
- _variableSizeData.position(_fixedSizeData.getInt());
- return _fixedSizeData.getInt();
+ protected int positionOffsetInVariableBufferAndGetLength(int rowId, int colId) {
+ int offset = getOffsetInFixedBuffer(rowId, colId);
+ _variableSizeData.position(_fixedSizeData.getInt(offset));
+ return _fixedSizeData.getInt(offset + 4);
}
@Override
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/MetadataBlock.java b/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/MetadataBlock.java
index 7ad53f6bf5..b100425e5a 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/MetadataBlock.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/MetadataBlock.java
@@ -44,12 +44,12 @@ public class MetadataBlock extends BaseDataBlock {
}
@Override
- protected void positionCursorInFixSizedBuffer(int rowId, int colId) {
+ protected int getOffsetInFixedBuffer(int rowId, int colId) {
throw new UnsupportedOperationException();
}
@Override
- protected int positionCursorInVariableBuffer(int rowId, int colId) {
+ protected int positionOffsetInVariableBufferAndGetLength(int rowId, int colId) {
throw new UnsupportedOperationException();
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/RowDataBlock.java b/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/RowDataBlock.java
index 7946cc428d..eca5912c38 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/RowDataBlock.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/RowDataBlock.java
@@ -85,16 +85,15 @@ public class RowDataBlock extends BaseDataBlock {
}
@Override
- protected void positionCursorInFixSizedBuffer(int rowId, int colId) {
- int position = rowId * _rowSizeInBytes + _columnOffsets[colId];
- _fixedSizeData.position(position);
+ protected int getOffsetInFixedBuffer(int rowId, int colId) {
+ return rowId * _rowSizeInBytes + _columnOffsets[colId];
}
@Override
- protected int positionCursorInVariableBuffer(int rowId, int colId) {
- positionCursorInFixSizedBuffer(rowId, colId);
- _variableSizeData.position(_fixedSizeData.getInt());
- return _fixedSizeData.getInt();
+ protected int positionOffsetInVariableBufferAndGetLength(int rowId, int colId) {
+ int offset = getOffsetInFixedBuffer(rowId, colId);
+ _variableSizeData.position(_fixedSizeData.getInt(offset));
+ return _fixedSizeData.getInt(offset + 4);
}
@Override
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/BaseDataTable.java b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/BaseDataTable.java
index a54079eb66..b4f01514a1 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/BaseDataTable.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/BaseDataTable.java
@@ -158,26 +158,22 @@ public abstract class BaseDataTable implements DataTable {
@Override
public int getInt(int rowId, int colId) {
- _fixedSizeData.position(rowId * _rowSizeInBytes + _columnOffsets[colId]);
- return _fixedSizeData.getInt();
+ return _fixedSizeData.getInt(rowId * _rowSizeInBytes + _columnOffsets[colId]);
}
@Override
public long getLong(int rowId, int colId) {
- _fixedSizeData.position(rowId * _rowSizeInBytes + _columnOffsets[colId]);
- return _fixedSizeData.getLong();
+ return _fixedSizeData.getLong(rowId * _rowSizeInBytes + _columnOffsets[colId]);
}
@Override
public float getFloat(int rowId, int colId) {
- _fixedSizeData.position(rowId * _rowSizeInBytes + _columnOffsets[colId]);
- return _fixedSizeData.getFloat();
+ return _fixedSizeData.getFloat(rowId * _rowSizeInBytes + _columnOffsets[colId]);
}
@Override
public double getDouble(int rowId, int colId) {
- _fixedSizeData.position(rowId * _rowSizeInBytes + _columnOffsets[colId]);
- return _fixedSizeData.getDouble();
+ return _fixedSizeData.getDouble(rowId * _rowSizeInBytes + _columnOffsets[colId]);
}
@Override
@@ -190,8 +186,7 @@ public abstract class BaseDataTable implements DataTable {
@Override
public String getString(int rowId, int colId) {
- _fixedSizeData.position(rowId * _rowSizeInBytes + _columnOffsets[colId]);
- int dictId = _fixedSizeData.getInt();
+ int dictId = _fixedSizeData.getInt(rowId * _rowSizeInBytes + _columnOffsets[colId]);
return _dictionaryMap.get(_dataSchema.getColumnName(colId)).get(dictId);
}
@@ -271,9 +266,9 @@ public abstract class BaseDataTable implements DataTable {
}
private int positionCursorInVariableBuffer(int rowId, int colId) {
- _fixedSizeData.position(rowId * _rowSizeInBytes + _columnOffsets[colId]);
- _variableSizeData.position(_fixedSizeData.getInt());
- return _fixedSizeData.getInt();
+ int offset = rowId * _rowSizeInBytes + _columnOffsets[colId];
+ _variableSizeData.position(_fixedSizeData.getInt(offset));
+ return _fixedSizeData.getInt(offset + 4);
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org