You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2019/09/04 22:40:51 UTC
[incubator-pinot] branch fixing_raw_bytes_comparison updated
(81438f1 -> bc76af5)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch fixing_raw_bytes_comparison
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard 81438f1 Adding more tests
discard fbfaa98 Support predicates on raw bytes column without dictionary
add 05609e4 Use Apache Pinot License (#4584)
new e8c4e08 Support predicates on raw bytes column without dictionary
new bc76af5 Adding more tests
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (81438f1)
\
N -- N -- N refs/heads/fixing_raw_bytes_comparison (bc76af5)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/02: Support predicates on raw bytes column
without dictionary
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch fixing_raw_bytes_comparison
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit e8c4e08190f39aa461db2ff92229447eed36fcb0
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Wed Sep 4 14:42:52 2019 -0700
Support predicates on raw bytes column without dictionary
---
.../BaseDictionaryBasedPredicateEvaluator.java | 10 +++++
.../BaseRawValueBasedPredicateEvaluator.java | 26 +++++++++++++
.../predicate/EqualsPredicateEvaluatorFactory.java | 22 +++++++++++
.../predicate/InPredicateEvaluatorFactory.java | 23 ++++++++++++
.../NotEqualsPredicateEvaluatorFactory.java | 22 +++++++++++
.../predicate/NotInPredicateEvaluatorFactory.java | 24 ++++++++++++
.../filter/predicate/PredicateEvaluator.java | 17 +++++++++
.../predicate/RangePredicateEvaluatorFactory.java | 43 ++++++++++++++++++++++
8 files changed, 187 insertions(+)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseDictionaryBasedPredicateEvaluator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseDictionaryBasedPredicateEvaluator.java
index 0d5c6c7..0f55f7b 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseDictionaryBasedPredicateEvaluator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseDictionaryBasedPredicateEvaluator.java
@@ -80,6 +80,16 @@ public abstract class BaseDictionaryBasedPredicateEvaluator extends BasePredicat
throw new UnsupportedOperationException();
}
+ @Override
+ public final boolean applySV(byte[] value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public final boolean applyMV(byte[][] values, int length) {
+ throw new UnsupportedOperationException();
+ }
+
// NOTE: override it for exclusive predicate
@Override
public int[] getNonMatchingDictIds() {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseRawValueBasedPredicateEvaluator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseRawValueBasedPredicateEvaluator.java
index 13dfc64..41be977 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseRawValueBasedPredicateEvaluator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/BaseRawValueBasedPredicateEvaluator.java
@@ -188,4 +188,30 @@ public abstract class BaseRawValueBasedPredicateEvaluator extends BasePredicateE
return false;
}
}
+
+ @Override
+ public boolean applySV(byte[] value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @SuppressWarnings("Duplicates")
+ @Override
+ public boolean applyMV(byte[][] values, int length) {
+ if (isExclusive()) {
+ for (int i = 0; i < length; i++) {
+ if (!applySV(values[i])) {
+ return false;
+ }
+ }
+ return true;
+ } else {
+ for (int i = 0; i < length; i++) {
+ if (applySV(values[i])) {
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java
index c4c35fd..7cee99c 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/EqualsPredicateEvaluatorFactory.java
@@ -19,6 +19,8 @@
package org.apache.pinot.core.operator.filter.predicate;
import org.apache.pinot.common.data.FieldSpec;
+import org.apache.pinot.common.utils.BytesUtils;
+import org.apache.pinot.common.utils.primitive.ByteArray;
import org.apache.pinot.core.common.Predicate;
import org.apache.pinot.core.common.predicate.EqPredicate;
import org.apache.pinot.core.segment.index.readers.Dictionary;
@@ -63,6 +65,8 @@ public class EqualsPredicateEvaluatorFactory {
return new DoubleRawValueBasedEqPredicateEvaluator(eqPredicate);
case STRING:
return new StringRawValueBasedEqPredicateEvaluator(eqPredicate);
+ case BYTES:
+ return new BytesRawValueBasedEqPredicateEvaluator(eqPredicate);
default:
throw new UnsupportedOperationException("Unsupported data type: " + dataType);
}
@@ -190,4 +194,22 @@ public class EqualsPredicateEvaluatorFactory {
return _matchingValue.equals(value);
}
}
+
+ private static final class BytesRawValueBasedEqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ final byte[] _matchingValue;
+
+ BytesRawValueBasedEqPredicateEvaluator(EqPredicate eqPredicate) {
+ _matchingValue = BytesUtils.toBytes(eqPredicate.getEqualsValue());
+ }
+
+ @Override
+ public Predicate.Type getPredicateType() {
+ return Predicate.Type.EQ;
+ }
+
+ @Override
+ public boolean applySV(byte[] value) {
+ return ByteArray.compare(_matchingValue, value) == 0;
+ }
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/InPredicateEvaluatorFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/InPredicateEvaluatorFactory.java
index 02a449d..e0d77df 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/InPredicateEvaluatorFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/InPredicateEvaluatorFactory.java
@@ -30,6 +30,7 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.apache.pinot.common.data.FieldSpec;
+import org.apache.pinot.common.utils.BytesUtils;
import org.apache.pinot.common.utils.HashUtil;
import org.apache.pinot.core.common.Predicate;
import org.apache.pinot.core.common.predicate.InPredicate;
@@ -75,6 +76,8 @@ public class InPredicateEvaluatorFactory {
return new DoubleRawValueBasedInPredicateEvaluator(inPredicate);
case STRING:
return new StringRawValueBasedInPredicateEvaluator(inPredicate);
+ case BYTES:
+ return new BytesRawValueBasedInPredicateEvaluator(inPredicate);
default:
throw new UnsupportedOperationException("Unsupported data type: " + dataType);
}
@@ -233,4 +236,24 @@ public class InPredicateEvaluatorFactory {
return _matchingValues.contains(value);
}
}
+
+ private static final class BytesRawValueBasedInPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ final Set<String> _matchingValues;
+
+ BytesRawValueBasedInPredicateEvaluator(InPredicate inPredicate) {
+ String[] values = inPredicate.getValues();
+ _matchingValues = new HashSet<>(HashUtil.getMinHashSetSize(values.length));
+ Collections.addAll(_matchingValues, values);
+ }
+
+ @Override
+ public Predicate.Type getPredicateType() {
+ return Predicate.Type.IN;
+ }
+
+ @Override
+ public boolean applySV(byte[] value) {
+ return _matchingValues.contains(BytesUtils.toHexString(value));
+ }
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/NotEqualsPredicateEvaluatorFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/NotEqualsPredicateEvaluatorFactory.java
index 514e2e2..4a92569 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/NotEqualsPredicateEvaluatorFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/NotEqualsPredicateEvaluatorFactory.java
@@ -19,6 +19,8 @@
package org.apache.pinot.core.operator.filter.predicate;
import org.apache.pinot.common.data.FieldSpec;
+import org.apache.pinot.common.utils.BytesUtils;
+import org.apache.pinot.common.utils.primitive.ByteArray;
import org.apache.pinot.core.common.Predicate;
import org.apache.pinot.core.common.predicate.NEqPredicate;
import org.apache.pinot.core.segment.index.readers.Dictionary;
@@ -63,6 +65,8 @@ public class NotEqualsPredicateEvaluatorFactory {
return new DoubleRawValueBasedNeqPredicateEvaluator(nEqPredicate);
case STRING:
return new StringRawValueBasedNeqPredicateEvaluator(nEqPredicate);
+ case BYTES:
+ return new BytesRawValueBasedNeqPredicateEvaluator(nEqPredicate);
default:
throw new UnsupportedOperationException("Unsupported data type: " + dataType);
}
@@ -215,4 +219,22 @@ public class NotEqualsPredicateEvaluatorFactory {
return !_nonMatchingValue.equals(value);
}
}
+
+ private static final class BytesRawValueBasedNeqPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ final byte[] _nonMatchingValue;
+
+ BytesRawValueBasedNeqPredicateEvaluator(NEqPredicate nEqPredicate) {
+ _nonMatchingValue = BytesUtils.toBytes(nEqPredicate.getNotEqualsValue());
+ }
+
+ @Override
+ public Predicate.Type getPredicateType() {
+ return Predicate.Type.NEQ;
+ }
+
+ @Override
+ public boolean applySV(byte[] value) {
+ return ByteArray.compare(_nonMatchingValue, value) != 0;
+ }
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/NotInPredicateEvaluatorFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/NotInPredicateEvaluatorFactory.java
index 33d31c1..678cd11 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/NotInPredicateEvaluatorFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/NotInPredicateEvaluatorFactory.java
@@ -30,6 +30,7 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.apache.pinot.common.data.FieldSpec;
+import org.apache.pinot.common.utils.BytesUtils;
import org.apache.pinot.common.utils.HashUtil;
import org.apache.pinot.core.common.Predicate;
import org.apache.pinot.core.common.predicate.NotInPredicate;
@@ -75,6 +76,8 @@ public class NotInPredicateEvaluatorFactory {
return new DoubleRawValueBasedNotInPredicateEvaluator(notInPredicate);
case STRING:
return new StringRawValueBasedNotInPredicateEvaluator(notInPredicate);
+ case BYTES:
+ return new BytesRawValueBasedNotInPredicateEvaluator(notInPredicate);
default:
throw new UnsupportedOperationException("Unsupported data type: " + dataType);
}
@@ -251,4 +254,25 @@ public class NotInPredicateEvaluatorFactory {
return !_nonMatchingValues.contains(value);
}
}
+
+ private static final class BytesRawValueBasedNotInPredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ final Set<String> _nonMatchingValues;
+
+ BytesRawValueBasedNotInPredicateEvaluator(NotInPredicate notInPredicate) {
+ String[] values = notInPredicate.getValues();
+ _nonMatchingValues = new HashSet<>(HashUtil.getMinHashSetSize(values.length));
+ Collections.addAll(_nonMatchingValues, values);
+ }
+
+ @Override
+ public Predicate.Type getPredicateType() {
+ return Predicate.Type.NOT_IN;
+ }
+
+ @Override
+ public boolean applySV(byte[] value) {
+ return !_nonMatchingValues.contains(BytesUtils.toHexString(value));
+ }
+ }
+
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/PredicateEvaluator.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/PredicateEvaluator.java
index 2255a14..9cb2894 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/PredicateEvaluator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/PredicateEvaluator.java
@@ -164,4 +164,21 @@ public interface PredicateEvaluator {
* @return Whether the entry matches the predicate
*/
boolean applyMV(String[] values, int length);
+
+ /**
+ * Apply a single-value entry to the predicate.
+ *
+ * @param value Raw value
+ * @return Whether the entry matches the predicate
+ */
+ boolean applySV(byte[] value);
+
+ /**
+ * Apply a multi-value entry to the predicate.
+ *
+ * @param values Array of raw values
+ * @param length Number of values in the entry
+ * @return Whether the entry matches the predicate
+ */
+ boolean applyMV(byte[][] values, int length);
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
index 4431fdc..f2b7ad3 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
@@ -21,6 +21,8 @@ package org.apache.pinot.core.operator.filter.predicate;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntSet;
import org.apache.pinot.common.data.FieldSpec;
+import org.apache.pinot.common.utils.BytesUtils;
+import org.apache.pinot.common.utils.primitive.ByteArray;
import org.apache.pinot.core.common.Predicate;
import org.apache.pinot.core.common.predicate.RangePredicate;
import org.apache.pinot.core.realtime.impl.dictionary.MutableDictionary;
@@ -71,6 +73,8 @@ public class RangePredicateEvaluatorFactory {
return new DoubleRawValueBasedRangePredicateEvaluator(rangePredicate);
case STRING:
return new StringRawValueBasedRangePredicateEvaluator(rangePredicate);
+ case BYTES:
+ return new BytesRawValueBasedRangePredicateEvaluator(rangePredicate);
default:
throw new UnsupportedOperationException("Unsupported data type: " + dataType);
}
@@ -408,4 +412,43 @@ public class RangePredicateEvaluatorFactory {
return result;
}
}
+
+ private static final class BytesRawValueBasedRangePredicateEvaluator extends BaseRawValueBasedPredicateEvaluator {
+ final byte[] _lowerBoundary;
+ final byte[] _upperBoundary;
+ final boolean _includeLowerBoundary;
+ final boolean _includeUpperBoundary;
+
+ BytesRawValueBasedRangePredicateEvaluator(RangePredicate rangePredicate) {
+ _lowerBoundary = BytesUtils.toBytes(rangePredicate.getLowerBoundary());
+ _upperBoundary = BytesUtils.toBytes(rangePredicate.getUpperBoundary());
+ _includeLowerBoundary = rangePredicate.includeLowerBoundary();
+ _includeUpperBoundary = rangePredicate.includeUpperBoundary();
+ }
+
+ @Override
+ public Predicate.Type getPredicateType() {
+ return Predicate.Type.RANGE;
+ }
+
+ @Override
+ public boolean applySV(byte[] value) {
+ boolean result = true;
+ if (!_lowerBoundary.equals("*")) {
+ if (_includeLowerBoundary) {
+ result = ByteArray.compare(_lowerBoundary, value) <= 0;
+ } else {
+ result = ByteArray.compare(_lowerBoundary, value) < 0;
+ }
+ }
+ if (!_upperBoundary.equals("*")) {
+ if (_includeUpperBoundary) {
+ result &= ByteArray.compare(_upperBoundary, value) >= 0;
+ } else {
+ result &= ByteArray.compare(_upperBoundary, value) > 0;
+ }
+ }
+ return result;
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 02/02: Adding more tests
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch fixing_raw_bytes_comparison
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit bc76af5b63312984a67bee2e856843290d536b4e
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Wed Sep 4 15:34:47 2019 -0700
Adding more tests
---
.../predicate/RangePredicateEvaluatorFactory.java | 16 ++++--
.../NoDictionaryEqualsPredicateEvaluatorsTest.java | 37 ++++++++++++++
.../NoDictionaryInPredicateEvaluatorTest.java | 40 +++++++++++++++
.../NoDictionaryRangePredicateEvaluatorTest.java | 57 ++++++++++++++++++++++
.../predicate/PredicateEvaluatorTestUtils.java | 7 +++
5 files changed, 153 insertions(+), 4 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
index f2b7ad3..d9ab7e7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RangePredicateEvaluatorFactory.java
@@ -420,8 +420,16 @@ public class RangePredicateEvaluatorFactory {
final boolean _includeUpperBoundary;
BytesRawValueBasedRangePredicateEvaluator(RangePredicate rangePredicate) {
- _lowerBoundary = BytesUtils.toBytes(rangePredicate.getLowerBoundary());
- _upperBoundary = BytesUtils.toBytes(rangePredicate.getUpperBoundary());
+ if (!"*".equals(rangePredicate.getLowerBoundary())) {
+ _lowerBoundary = BytesUtils.toBytes(rangePredicate.getLowerBoundary());
+ } else {
+ _lowerBoundary = null;
+ }
+ if (!"*".equals(rangePredicate.getUpperBoundary())) {
+ _upperBoundary = BytesUtils.toBytes(rangePredicate.getUpperBoundary());
+ } else {
+ _upperBoundary = null;
+ }
_includeLowerBoundary = rangePredicate.includeLowerBoundary();
_includeUpperBoundary = rangePredicate.includeUpperBoundary();
}
@@ -434,14 +442,14 @@ public class RangePredicateEvaluatorFactory {
@Override
public boolean applySV(byte[] value) {
boolean result = true;
- if (!_lowerBoundary.equals("*")) {
+ if (_lowerBoundary != null) {
if (_includeLowerBoundary) {
result = ByteArray.compare(_lowerBoundary, value) <= 0;
} else {
result = ByteArray.compare(_lowerBoundary, value) < 0;
}
}
- if (!_upperBoundary.equals("*")) {
+ if (_upperBoundary != null) {
if (_includeUpperBoundary) {
result &= ByteArray.compare(_upperBoundary, value) >= 0;
} else {
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryEqualsPredicateEvaluatorsTest.java b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryEqualsPredicateEvaluatorsTest.java
index 31b0220..823cdc3 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryEqualsPredicateEvaluatorsTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryEqualsPredicateEvaluatorsTest.java
@@ -24,6 +24,7 @@ import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.pinot.common.data.FieldSpec;
+import org.apache.pinot.common.utils.BytesUtils;
import org.apache.pinot.core.common.predicate.EqPredicate;
import org.apache.pinot.core.common.predicate.NEqPredicate;
import org.apache.pinot.core.operator.filter.predicate.EqualsPredicateEvaluatorFactory;
@@ -219,4 +220,40 @@ public class NoDictionaryEqualsPredicateEvaluatorsTest {
!ArrayUtils.contains(randomStrings, stringValue));
}
}
+
+ @Test
+ public void testBytesPredicateEvaluators() {
+ byte[] bytesValue = RandomStringUtils.random(MAX_STRING_LENGTH).getBytes();
+ String hexStringValue = BytesUtils.toHexString(bytesValue);
+ EqPredicate eqPredicate = new EqPredicate(COLUMN_NAME, Collections.singletonList(hexStringValue));
+ PredicateEvaluator eqPredicateEvaluator =
+ EqualsPredicateEvaluatorFactory.newRawValueBasedEvaluator(eqPredicate, FieldSpec.DataType.BYTES);
+
+ NEqPredicate neqPredicate = new NEqPredicate(COLUMN_NAME, Collections.singletonList(hexStringValue));
+ PredicateEvaluator neqPredicateEvaluator =
+ NotEqualsPredicateEvaluatorFactory.newRawValueBasedEvaluator(neqPredicate, FieldSpec.DataType.BYTES);
+
+ Assert.assertTrue(eqPredicateEvaluator.applySV(bytesValue));
+ Assert.assertFalse(neqPredicateEvaluator.applySV(bytesValue));
+
+ byte[][] randomBytesArray = new byte[NUM_MULTI_VALUES][];
+ PredicateEvaluatorTestUtils.fillRandom(randomBytesArray, MAX_STRING_LENGTH);
+ randomBytesArray[_random.nextInt(NUM_MULTI_VALUES)] = bytesValue;
+
+ Assert.assertTrue(eqPredicateEvaluator.applyMV(randomBytesArray, NUM_MULTI_VALUES));
+ Assert.assertFalse(neqPredicateEvaluator.applyMV(randomBytesArray, NUM_MULTI_VALUES));
+
+ for (int i = 0; i < 100; i++) {
+ byte[] randomBytes = RandomStringUtils.random(MAX_STRING_LENGTH).getBytes();
+ String randomString = BytesUtils.toHexString(randomBytes);
+ Assert.assertEquals(eqPredicateEvaluator.applySV(randomBytes), (randomString.equals(hexStringValue)));
+ Assert.assertEquals(neqPredicateEvaluator.applySV(randomBytes), (!randomString.equals(hexStringValue)));
+
+ PredicateEvaluatorTestUtils.fillRandom(randomBytesArray, MAX_STRING_LENGTH);
+ Assert.assertEquals(eqPredicateEvaluator.applyMV(randomBytesArray, NUM_MULTI_VALUES),
+ ArrayUtils.contains(randomBytesArray, hexStringValue));
+ Assert.assertEquals(neqPredicateEvaluator.applyMV(randomBytesArray, NUM_MULTI_VALUES),
+ !ArrayUtils.contains(randomBytesArray, hexStringValue));
+ }
+ }
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryInPredicateEvaluatorTest.java b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryInPredicateEvaluatorTest.java
index 217f186..9435aaf 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryInPredicateEvaluatorTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryInPredicateEvaluatorTest.java
@@ -34,6 +34,7 @@ import java.util.Set;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.pinot.common.data.FieldSpec;
+import org.apache.pinot.common.utils.BytesUtils;
import org.apache.pinot.core.common.predicate.InPredicate;
import org.apache.pinot.core.common.predicate.NotInPredicate;
import org.apache.pinot.core.operator.filter.predicate.InPredicateEvaluatorFactory;
@@ -252,4 +253,43 @@ public class NoDictionaryInPredicateEvaluatorTest {
Assert.assertTrue(inPredicateEvaluator.applyMV(multiValues, NUM_MULTI_VALUES));
Assert.assertFalse(notInPredicateEvaluator.applyMV(multiValues, NUM_MULTI_VALUES));
}
+
+ @Test
+ public void testBytesPredicateEvaluators() {
+ List<String> stringValues = new ArrayList<>(NUM_PREDICATE_VALUES);
+ Set<byte[]> valueSet = new HashSet<>();
+
+ for (int i = 0; i < 100; i++) {
+ byte[] value = RandomStringUtils.random(MAX_STRING_LENGTH).getBytes();
+ valueSet.add(value);
+ stringValues.add(BytesUtils.toHexString(value));
+ }
+
+ InPredicate inPredicate = new InPredicate(COLUMN_NAME, stringValues);
+ PredicateEvaluator inPredicateEvaluator =
+ InPredicateEvaluatorFactory.newRawValueBasedEvaluator(inPredicate, FieldSpec.DataType.BYTES);
+
+ NotInPredicate notInPredicate = new NotInPredicate(COLUMN_NAME, stringValues);
+ PredicateEvaluator notInPredicateEvaluator =
+ NotInPredicateEvaluatorFactory.newRawValueBasedEvaluator(notInPredicate, FieldSpec.DataType.BYTES);
+
+ for (byte[] value : valueSet) {
+ Assert.assertTrue(inPredicateEvaluator.applySV(value));
+ Assert.assertFalse(notInPredicateEvaluator.applySV(value));
+ }
+
+ for (int i = 0; i < 100; i++) {
+ byte[] value = RandomStringUtils.random(MAX_STRING_LENGTH).getBytes();
+ Assert.assertEquals(inPredicateEvaluator.applySV(value), valueSet.contains(value));
+ Assert.assertEquals(notInPredicateEvaluator.applySV(value), !valueSet.contains(value));
+ }
+
+ byte[][] multiValues = new byte[NUM_MULTI_VALUES][];
+ PredicateEvaluatorTestUtils.fillRandom(multiValues, MAX_STRING_LENGTH);
+ multiValues[_random.nextInt(NUM_MULTI_VALUES)] =
+ BytesUtils.toBytes(stringValues.get(_random.nextInt(NUM_PREDICATE_VALUES)));
+
+ Assert.assertTrue(inPredicateEvaluator.applyMV(multiValues, NUM_MULTI_VALUES));
+ Assert.assertFalse(notInPredicateEvaluator.applyMV(multiValues, NUM_MULTI_VALUES));
+ }
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryRangePredicateEvaluatorTest.java b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryRangePredicateEvaluatorTest.java
index 5f68646..94887a2 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryRangePredicateEvaluatorTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/predicate/NoDictionaryRangePredicateEvaluatorTest.java
@@ -20,6 +20,8 @@ package org.apache.pinot.core.predicate;
import java.util.Collections;
import org.apache.pinot.common.data.FieldSpec;
+import org.apache.pinot.common.utils.BytesUtils;
+import org.apache.pinot.common.utils.primitive.ByteArray;
import org.apache.pinot.core.common.predicate.RangePredicate;
import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator;
import org.apache.pinot.core.operator.filter.predicate.RangePredicateEvaluatorFactory;
@@ -261,6 +263,61 @@ public class NoDictionaryRangePredicateEvaluatorTest {
}
}
+ @Test
+ public void testBytesPredicateEvaluator() {
+
+ PredicateEvaluator predicateEvaluator = buildRangePredicate("[10\t\t20]", FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertEquals(predicateEvaluator.applySV(value),
+ (ByteArray.compare(value, new byte[]{0x10}) >= 0 && ByteArray.compare(value, new byte[]{0x20}) <= 0));
+ }
+
+ predicateEvaluator = buildRangePredicate("(10\t\t20]", FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertEquals(predicateEvaluator.applySV(value),
+ (ByteArray.compare(value, new byte[]{0x10}) > 0 && ByteArray.compare(value, new byte[]{0x20}) <= 0));
+ }
+
+ predicateEvaluator = buildRangePredicate("(10\t\t20)", FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertEquals(predicateEvaluator.applySV(value),
+ (ByteArray.compare(value, new byte[]{0x10}) > 0 && ByteArray.compare(value, new byte[]{0x20}) < 0));
+ }
+
+ predicateEvaluator = buildRangePredicate("(*\t\t10]", FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertEquals(predicateEvaluator.applySV(value), ByteArray.compare(value, new byte[]{0x10}) <= 0);
+ }
+
+ predicateEvaluator = buildRangePredicate("(*\t\t10)", FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertEquals(predicateEvaluator.applySV(value), ByteArray.compare(value, new byte[]{0x10}) < 0);
+ }
+
+ predicateEvaluator = buildRangePredicate("[10\t\t*)", FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertEquals(predicateEvaluator.applySV(value), ByteArray.compare(value, new byte[]{0x10}) >= 0);
+ }
+
+ predicateEvaluator = buildRangePredicate("(10\t\t*)", FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertEquals(predicateEvaluator.applySV(value), ByteArray.compare(value, new byte[]{0x10}) > 0);
+ }
+
+ predicateEvaluator = buildRangePredicate("(*\t\t*)", FieldSpec.DataType.BYTES);
+ for (int i = 0x00; i < 0x30; i++) {
+ byte[] value = Integer.toString(i).getBytes();
+ Assert.assertTrue(predicateEvaluator.applySV(value));
+ }
+ }
+
private PredicateEvaluator buildRangePredicate(String rangeString, FieldSpec.DataType dataType) {
RangePredicate predicate = new RangePredicate(COLUMN_NAME, Collections.singletonList(rangeString));
return RangePredicateEvaluatorFactory.newRawValueBasedEvaluator(predicate, dataType);
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/predicate/PredicateEvaluatorTestUtils.java b/pinot-core/src/test/java/org/apache/pinot/core/predicate/PredicateEvaluatorTestUtils.java
index 6af6e44..7f33f09 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/predicate/PredicateEvaluatorTestUtils.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/predicate/PredicateEvaluatorTestUtils.java
@@ -20,6 +20,7 @@ package org.apache.pinot.core.predicate;
import java.util.Random;
import org.apache.commons.lang.RandomStringUtils;
+import org.apache.pinot.common.utils.BytesUtils;
public class PredicateEvaluatorTestUtils {
@@ -54,4 +55,10 @@ public class PredicateEvaluatorTestUtils {
randomValues[i] = RandomStringUtils.random(maxStringLength);
}
}
+
+ public static void fillRandom(byte[][] randomValues, int maxStringLength) {
+ for (int i = 0; i < randomValues.length; i++) {
+ randomValues[i] = RandomStringUtils.random(maxStringLength).getBytes();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org