You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2019/04/01 21:56:09 UTC
[incubator-pinot] branch master updated: Use hex string as the
representation of byte array for queries (#4041)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 202009f Use hex string as the representation of byte array for queries (#4041)
202009f is described below
commit 202009f973d64670f2a480c2546e4a7bb8f06e8a
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Mon Apr 1 14:56:04 2019 -0700
Use hex string as the representation of byte array for queries (#4041)
* Use hex string as the representation of byte array for queries
* Adding test for indexing hexstring
* Address comments
---
.../dictionary/BytesOffHeapMutableDictionary.java | 33 +++++++++++++++++---
.../dictionary/BytesOnHeapMutableDictionary.java | 35 +++++++++++++++++++---
.../impl/dictionary/MutableDictionaryTest.java | 4 ++-
3 files changed, 63 insertions(+), 9 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/dictionary/BytesOffHeapMutableDictionary.java b/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/dictionary/BytesOffHeapMutableDictionary.java
index 816e0de..5220749 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/dictionary/BytesOffHeapMutableDictionary.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/dictionary/BytesOffHeapMutableDictionary.java
@@ -21,6 +21,9 @@ package org.apache.pinot.core.realtime.impl.dictionary;
import java.io.IOException;
import java.util.Arrays;
import javax.annotation.Nonnull;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.pinot.common.Utils;
import org.apache.pinot.common.utils.primitive.ByteArray;
import org.apache.pinot.core.io.readerwriter.PinotDataBufferMemoryManager;
import org.apache.pinot.core.io.writer.impl.MutableOffHeapByteArrayStore;
@@ -30,6 +33,7 @@ import org.apache.pinot.core.io.writer.impl.MutableOffHeapByteArrayStore;
* OffHeap mutable dictionary for Bytes data type.
*/
public class BytesOffHeapMutableDictionary extends BaseOffHeapMutableDictionary {
+
private final MutableOffHeapByteArrayStore _byteStore;
private ByteArray _min = null;
@@ -52,8 +56,19 @@ public class BytesOffHeapMutableDictionary extends BaseOffHeapMutableDictionary
@Override
public int indexOf(Object rawValue) {
- assert rawValue instanceof byte[];
- byte[] bytes = (byte[]) rawValue;
+ byte[] bytes = null;
+ // Convert hex string to byte[].
+ if (rawValue instanceof byte[]) {
+ bytes = (byte[]) rawValue;
+ } else if (rawValue instanceof String) {
+ try {
+ bytes = Hex.decodeHex(((String) rawValue).toCharArray());
+ } catch (DecoderException e) {
+ Utils.rethrowException(e);
+ }
+ } else {
+ assert rawValue instanceof byte[];
+ }
return getDictId(new ByteArray(bytes), bytes);
}
@@ -80,8 +95,18 @@ public class BytesOffHeapMutableDictionary extends BaseOffHeapMutableDictionary
@Override
public void index(@Nonnull Object rawValue) {
- assert rawValue instanceof byte[];
- byte[] bytes = (byte[]) rawValue;
+ byte[] bytes = null;
+ // Convert hex string to byte[].
+ if (rawValue instanceof String) {
+ try {
+ bytes = Hex.decodeHex(((String) rawValue).toCharArray());
+ } catch (DecoderException e) {
+ Utils.rethrowException(e);
+ }
+ } else {
+ assert rawValue instanceof byte[];
+ bytes = (byte[]) rawValue;
+ }
ByteArray byteArray = new ByteArray(bytes);
indexValue(byteArray, bytes);
updateMinMax(byteArray);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/dictionary/BytesOnHeapMutableDictionary.java b/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/dictionary/BytesOnHeapMutableDictionary.java
index df36098..37236c3 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/dictionary/BytesOnHeapMutableDictionary.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/dictionary/BytesOnHeapMutableDictionary.java
@@ -20,6 +20,9 @@ package org.apache.pinot.core.realtime.impl.dictionary;
import java.util.Arrays;
import javax.annotation.Nonnull;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.pinot.common.Utils;
import org.apache.pinot.common.utils.primitive.ByteArray;
@@ -27,13 +30,26 @@ import org.apache.pinot.common.utils.primitive.ByteArray;
* OnHeap mutable dictionary of Bytes type.
*/
public class BytesOnHeapMutableDictionary extends BaseOnHeapMutableDictionary {
+
private ByteArray _min = null;
private ByteArray _max = null;
@Override
public int indexOf(Object rawValue) {
- assert rawValue instanceof byte[];
- return getDictId(new ByteArray((byte[]) rawValue));
+ byte[] bytes = null;
+ // Convert hex string to byte[].
+ if (rawValue instanceof byte[]) {
+ bytes = (byte[]) rawValue;
+ } else if (rawValue instanceof String) {
+ try {
+ bytes = Hex.decodeHex(((String) rawValue).toCharArray());
+ } catch (DecoderException e) {
+ Utils.rethrowException(e);
+ }
+ } else {
+ assert rawValue instanceof byte[];
+ }
+ return getDictId(new ByteArray(bytes));
}
@Override
@@ -48,8 +64,19 @@ public class BytesOnHeapMutableDictionary extends BaseOnHeapMutableDictionary {
@Override
public void index(@Nonnull Object rawValue) {
- assert rawValue instanceof byte[];
- ByteArray byteArray = new ByteArray((byte[]) rawValue);
+ byte[] bytes = null;
+ // Convert hex string to byte[].
+ if (rawValue instanceof String) {
+ try {
+ bytes = Hex.decodeHex(((String) rawValue).toCharArray());
+ } catch (DecoderException e) {
+ Utils.rethrowException(e);
+ }
+ } else {
+ assert rawValue instanceof byte[];
+ bytes = (byte[]) rawValue;
+ }
+ ByteArray byteArray = new ByteArray(bytes);
indexValue(byteArray);
updateMinMax(byteArray);
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/dictionary/MutableDictionaryTest.java b/pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/dictionary/MutableDictionaryTest.java
index ce79b10..99a28d2 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/dictionary/MutableDictionaryTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/dictionary/MutableDictionaryTest.java
@@ -30,6 +30,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
+import org.apache.commons.codec.binary.Hex;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.pinot.common.data.FieldSpec;
import org.apache.pinot.common.utils.primitive.ByteArray;
@@ -174,7 +175,8 @@ public class MutableDictionaryTest {
Comparable value =
(i == 0 && dataType == FieldSpec.DataType.INT) ? Integer.MIN_VALUE : makeRandomObjectOfType(dataType);
- Object rawValue = dataType == FieldSpec.DataType.BYTES ? ((ByteArray) value).getBytes() : value;
+ Object rawValue = dataType == FieldSpec.DataType.BYTES ? ((i % 2 == 0) ? ((ByteArray) value).getBytes()
+ : Hex.encodeHexString(((ByteArray) value).getBytes())) : value;
if (valueToDictId.containsKey(value)) {
Assert.assertEquals(dictionary.indexOf(rawValue), (int) valueToDictId.get(value));
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org