You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2019/04/17 00:56:37 UTC
[incubator-pinot] branch improve-to-bytes-method created (now
f15c547)
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a change to branch improve-to-bytes-method
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at f15c547 Declare byte array size when initializing ByteArrayOutputStream
This branch includes the following new commits:
new f15c547 Declare byte array size when initializing ByteArrayOutputStream
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Declare byte array size when initializing
ByteArrayOutputStream
Posted by jl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch improve-to-bytes-method
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit f15c5476f6066f110d8867e626e3f2f2c120d2c8
Author: jackjlli <jl...@linkedin.com>
AuthorDate: Tue Apr 16 17:56:15 2019 -0700
Declare byte array size when initializing ByteArrayOutputStream
---
.../core/common/datatable/DataTableImplV2.java | 31 +++++++++++++++++-----
1 file changed, 25 insertions(+), 6 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV2.java b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV2.java
index c2abbe9..b838c86 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV2.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/datatable/DataTableImplV2.java
@@ -232,7 +232,20 @@ public class DataTableImplV2 implements DataTable {
@Override
public byte[] toBytes()
throws IOException {
- ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ byte[] metadataBytes = serializeMetadata();
+
+ byte[] dictionaryMapBytes = null;
+ if (_dictionaryMap != null) {
+ dictionaryMapBytes = serializeDictionaryMap();
+ }
+
+ byte[] dataSchemaBytes = null;
+ if (_dataSchema != null) {
+ dataSchemaBytes = _dataSchema.toBytes();
+ }
+
+ int totalSize = getTotalByteArraySize(metadataBytes, dictionaryMapBytes, dataSchemaBytes);
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(totalSize);
DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
dataOutputStream.writeInt(VERSION);
dataOutputStream.writeInt(_numRows);
@@ -241,9 +254,7 @@ public class DataTableImplV2 implements DataTable {
// Write dictionary.
dataOutputStream.writeInt(dataOffset);
- byte[] dictionaryMapBytes = null;
if (_dictionaryMap != null) {
- dictionaryMapBytes = serializeDictionaryMap();
dataOutputStream.writeInt(dictionaryMapBytes.length);
dataOffset += dictionaryMapBytes.length;
} else {
@@ -252,15 +263,12 @@ public class DataTableImplV2 implements DataTable {
// Write metadata.
dataOutputStream.writeInt(dataOffset);
- byte[] metadataBytes = serializeMetadata();
dataOutputStream.writeInt(metadataBytes.length);
dataOffset += metadataBytes.length;
// Write data schema.
dataOutputStream.writeInt(dataOffset);
- byte[] dataSchemaBytes = null;
if (_dataSchema != null) {
- dataSchemaBytes = _dataSchema.toBytes();
dataOutputStream.writeInt(dataSchemaBytes.length);
dataOffset += dataSchemaBytes.length;
} else {
@@ -346,6 +354,17 @@ public class DataTableImplV2 implements DataTable {
return byteArrayOutputStream.toByteArray();
}
+ private int getTotalByteArraySize(byte[] metadataBytes, byte[] dictionaryMapBytes, byte[] dataSchemaBytes) {
+ int metadataSize = metadataBytes.length + 1;
+ int dictionarySize = dictionaryMapBytes != null ? dictionaryMapBytes.length + 1 : 2;
+ int dataSchemaSize = dataSchemaBytes != null ? dataSchemaBytes.length + 1 : 2;
+ int fixedSizeDataSize = _fixedSizeDataBytes != null ? _fixedSizeDataBytes.length + 1 : 2;
+ int variableSizeDataSize = _variableSizeDataBytes != null ? _variableSizeDataBytes.length + 1 : 2;
+
+ // total size that byte array needs. 3 denotes version, number of rows and columns.
+ return 3 + metadataSize + dictionarySize + dataSchemaSize + fixedSizeDataSize + variableSizeDataSize;
+ }
+
@Nonnull
@Override
public Map<String, String> getMetadata() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org