You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by al...@apache.org on 2019/03/11 21:24:35 UTC
[asterixdb] branch master updated: [ASTERIXDB-2516][COMP] Change
logical comparators handling of null/missing
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 754a8b1 [ASTERIXDB-2516][COMP] Change logical comparators handling of null/missing
754a8b1 is described below
commit 754a8b18e2e6b946f2067c52c5bea137de215c14
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Sun Mar 10 15:46:51 2019 -0700
[ASTERIXDB-2516][COMP] Change logical comparators handling of null/missing
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Change the result of comparing against null and missing.
For arrays, comparing against null or missing item will result in incomparable.
For records, comparing against a null field will result in incomparable.
- modified test cases accordingly
- moved few object factories to the common place.
- modified compareRecords to avoid string construction for field names
- few code clean-ups.
Change-Id: Id93bea76e13658768e08a98fd373c71a901ceec5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3259
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
---
.../results/comparison/arrays/arrays.007.adm | 2 +-
.../results/comparison/arrays/arrays.014.adm | 1 -
.../results/comparison/arrays/arrays.021.adm | 2 +-
.../results/comparison/arrays/arrays.022.adm | 2 +-
.../results/comparison/records/records.005.adm | 2 +-
.../dataflow/data/nontagged/CompareHashUtil.java | 9 +-
.../AbstractAGenericBinaryComparator.java | 12 +-
.../LogicalComplexBinaryComparator.java | 137 +++++----------------
.../comparators/LogicalScalarBinaryComparator.java | 2 +-
.../hash/AMurmurHash3BinaryHashFunctionFamily.java | 8 +-
.../asterix/om/util/container/ObjectFactories.java | 15 ++-
11 files changed, 67 insertions(+), 125 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.007.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.007.adm
index d386253..e1b8022 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.007.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.007.adm
@@ -1 +1 @@
-{ "t1": { "c": "[[1.0,4], [5,9,11,14]] = [[1.0,4], [5,9,11,14]]", "r": true }, "t2": { "c": "[[5,2,7], ['green','black'], [date('2013-01-01')]] = [[5,2,7], ['green','black'], [date('2013-01-01')]]", "r": true }, "t3": { "c": "[['white','yellow','brown'], 6] != [['white','yellow','brown'], double('6')]", "r": false }, "t4": { "c": "[['white','yellow','brown'], 6] != [double('6'), ['white','yellow','brown']]", "r": null }, "t5": { "c": "[ [[1,2,3], 'gold', ['sql++', 5]], [tinyint('4'), tin [...]
\ No newline at end of file
+{ "t1": { "c": "[[1.0,4], [5,9,11,14]] = [[1.0,4], [5,9,11,14]]", "r": true }, "t2": { "c": "[[5,2,7], ['green','black'], [date('2013-01-01')]] = [[5,2,7], ['green','black'], [date('2013-01-01')]]", "r": true }, "t3": { "c": "[['white','yellow','brown'], 6] != [['white','yellow','brown'], double('6')]", "r": false }, "t4": { "c": "[['white','yellow','brown'], 6] != [double('6'), ['white','yellow','brown']]", "r": null }, "t5": { "c": "[ [[1,2,3], 'gold', ['sql++', 5]], [tinyint('4'), tin [...]
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.014.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.014.adm
index 6cfa561..a98f8bc 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.014.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.014.adm
@@ -1,2 +1 @@
-{ "id": 3, "array1": [ 2, 1 ], "OP": "< is_missing", "array2": [ null, 3, 3 ] }
{ "id": 7, "array1": [ 2, 1 ], "OP": "< is_missing" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.021.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.021.adm
index 55b9648..e628210 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.021.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.021.adm
@@ -1 +1 @@
-{ "t1": { "c": "[9,2] = null", "r": null }, "t2": { "c": "[9,2] = missing" }, "t3": { "c": "[9,2] > null", "r": null }, "t4": { "c": "[9,2] > missing" }, "t5": { "c": "['red', null] < ['red', null]", "r": null }, "t6": { "c": "[missing,2] < [null,3]" }, "t7": { "c": "[1,2] < [1,2,missing]", "r": true }, "t8": { "c": "[1,2] < [1,2,null]", "r": true }, "t9": { "c": "[null,5] >= [null,5]", "r": null }, "t10": { "c": "[null,8] < [4, 9]", "r": null }, "t11": { "c": "[1,2,missing] != [1,2,miss [...]
\ No newline at end of file
+{ "t1": { "c": "[9,2] = null", "r": null }, "t2": { "c": "[9,2] = missing" }, "t3": { "c": "[9,2] > null", "r": null }, "t4": { "c": "[9,2] > missing" }, "t5": { "c": "['red', null] < ['red', null]", "r": null }, "t6": { "c": "[missing,2] < [null,3]", "r": null }, "t7": { "c": "[1,2] < [1,2,missing]", "r": true }, "t8": { "c": "[1,2] < [1,2,null]", "r": true }, "t9": { "c": "[null,5] >= [null,5]", "r": null }, "t10": { "c": "[null,8] < [4, 9]", "r": null }, "t11": { "c": "[1,2,missing] ! [...]
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.022.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.022.adm
index 9fc20b1..6ec62e3 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.022.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/arrays/arrays.022.adm
@@ -1 +1 @@
-{ "t1": { "c": "[1, 'string'] != [2, 9]", "r": null }, "t2": { "c": "[1, 'string'] > [2, 9]", "r": null }, "t3": { "c": "[9, {'id': 2}] < [1, {'id': 3}]", "r": null }, "t4": { "c": "[1, 2] = ['string', 2, 3, 4]", "r": null }, "t5": { "c": "[null, 2, 3, 4, 5] = [1, 2]", "r": false }, "t6": { "c": "[1, null, 3] = [1, 2, 'string']", "r": null }, "t7": { "c": "[1, null] = [2, 5]", "r": false }, "t8": { "c": "[1, null, 3, 7] = [1, 2, 9, 5]", "r": false }, "t9": { "c": "[null, 'string'] < [1, [...]
\ No newline at end of file
+{ "t1": { "c": "[1, 'string'] != [2, 9]", "r": null }, "t2": { "c": "[1, 'string'] > [2, 9]", "r": null }, "t3": { "c": "[9, {'id': 2}] < [1, {'id': 3}]", "r": null }, "t4": { "c": "[1, 2] = ['string', 2, 3, 4]", "r": null }, "t5": { "c": "[null, 2, 3, 4, 5] = [1, 2]", "r": null }, "t6": { "c": "[1, null, 3] = [1, 2, 'string']", "r": null }, "t7": { "c": "[1, null] = [2, 5]", "r": null }, "t8": { "c": "[1, null, 3, 7] = [1, 2, 9, 5]", "r": null }, "t9": { "c": "[null, 'string'] < [1, 2]" [...]
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/records/records.005.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/records/records.005.adm
index 4b7aac1..be11304 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/records/records.005.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/comparison/records/records.005.adm
@@ -1 +1 @@
-{ "t1": { "c": "{'a': 2, 'b': null} = {'a': 2, 'b': 3}", "r": null }, "t2": { "c": "{'a': 2, 'b': missing} = {'a': 2, 'b': 3}", "r": false }, "t3": { "c": "{'list': [1, null], 'f': 3} = {'f': 3, 'list': [1, 2]}", "r": null }, "t4": { "c": "{'list': [1, missing], 'f': 3} = {'f': 3, 'list': [1, 2]}" }, "t5": { "c": "{'a': 4, 'b': null} = {'a': 2, 'b': 3}", "r": false } }
\ No newline at end of file
+{ "t1": { "c": "{'a': 2, 'b': null} = {'a': 2, 'b': 3}", "r": null }, "t2": { "c": "{'a': 2, 'b': missing} = {'a': 2, 'b': 3}", "r": false }, "t3": { "c": "{'list': [1, null], 'f': 3} = {'f': 3, 'list': [1, 2]}", "r": null }, "t4": { "c": "{'list': [1, missing], 'f': 3} = {'f': 3, 'list': [1, 2]}", "r": null }, "t5": { "c": "{'a': 4, 'b': null} = {'a': 2, 'b': 3}", "r": null } }
\ No newline at end of file
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/CompareHashUtil.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/CompareHashUtil.java
index 833dde1..a42e5aa 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/CompareHashUtil.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/CompareHashUtil.java
@@ -79,14 +79,11 @@ public class CompareHashUtil {
throw new IllegalStateException();
}
- public static IAType getType(ARecordType recordType, int fieldIdx, IVisitablePointable fieldValue)
- throws HyracksDataException {
+ public static IAType getType(ARecordType recordType, int fieldIdx, ATypeTag fieldTag) throws HyracksDataException {
IAType[] fieldTypes = recordType.getFieldTypes();
if (fieldIdx >= fieldTypes.length) {
- byte tag = fieldValue.getByteArray()[fieldValue.getStartOffset()];
- ATypeTag fieldRuntimeTag = VALUE_TYPE_MAPPING[tag];
- return fieldRuntimeTag.isDerivedType() ? DefaultOpenFieldType.getDefaultOpenFieldType(fieldRuntimeTag)
- : TypeTagUtil.getBuiltinTypeByTag(fieldRuntimeTag);
+ return fieldTag.isDerivedType() ? DefaultOpenFieldType.getDefaultOpenFieldType(fieldTag)
+ : TypeTagUtil.getBuiltinTypeByTag(fieldTag);
}
return fieldTypes[fieldIdx];
}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/AbstractAGenericBinaryComparator.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/AbstractAGenericBinaryComparator.java
index aa6a168..da50c56 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/AbstractAGenericBinaryComparator.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/AbstractAGenericBinaryComparator.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.dataflow.data.nontagged.comparators;
+import static org.apache.asterix.om.types.ATypeTag.VALUE_TYPE_MAPPING;
+
import java.io.IOException;
import java.util.Comparator;
import java.util.List;
@@ -113,7 +115,7 @@ abstract class AbstractAGenericBinaryComparator implements IBinaryComparator {
protected final IAType rightType;
// a storage to promote a value
private final ArrayBackedValueStorage castBuffer;
- private final IObjectPool<IMutableValueStorage, ATypeTag> storageAllocator;
+ private final IObjectPool<IMutableValueStorage, Void> storageAllocator;
private final IObjectPool<IPointable, Void> voidPointableAllocator;
// used for record comparison, sorting field names
private final PointableAllocator recordAllocator;
@@ -416,6 +418,7 @@ abstract class AbstractAGenericBinaryComparator implements IBinaryComparator {
int leftFieldIdx, rightFieldIdx;
IAType leftFieldType, rightFieldType;
IVisitablePointable leftFieldName, leftFieldValue, rightFieldName, rightFieldValue;
+ ATypeTag fieldTag;
while (!leftNamesHeap.isEmpty() && !rightNamesHeap.isEmpty()) {
leftFieldName = leftNamesHeap.poll();
rightFieldName = rightNamesHeap.poll();
@@ -431,9 +434,10 @@ abstract class AbstractAGenericBinaryComparator implements IBinaryComparator {
rightFieldIdx = CompareHashUtil.getIndex(rightFieldsNames, rightFieldName);
leftFieldValue = leftFieldsValues.get(leftFieldIdx);
rightFieldValue = rightFieldsValues.get(rightFieldIdx);
- leftFieldType = CompareHashUtil.getType(leftRecordType, leftFieldIdx, leftFieldValue);
- rightFieldType = CompareHashUtil.getType(rightRecordType, rightFieldIdx, rightFieldValue);
-
+ fieldTag = VALUE_TYPE_MAPPING[leftFieldValue.getByteArray()[leftFieldValue.getStartOffset()]];
+ leftFieldType = CompareHashUtil.getType(leftRecordType, leftFieldIdx, fieldTag);
+ fieldTag = VALUE_TYPE_MAPPING[rightFieldValue.getByteArray()[rightFieldValue.getStartOffset()]];
+ rightFieldType = CompareHashUtil.getType(rightRecordType, rightFieldIdx, fieldTag);
result = compare(leftFieldType, leftFieldValue.getByteArray(), leftFieldValue.getStartOffset(),
leftFieldValue.getLength(), rightFieldType, rightFieldValue.getByteArray(),
rightFieldValue.getStartOffset(), rightFieldValue.getLength());
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/LogicalComplexBinaryComparator.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/LogicalComplexBinaryComparator.java
index 87d21da..a5924b9 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/LogicalComplexBinaryComparator.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/LogicalComplexBinaryComparator.java
@@ -20,19 +20,21 @@ package org.apache.asterix.dataflow.data.nontagged.comparators;
import static org.apache.asterix.om.types.ATypeTag.SERIALIZED_MISSING_TYPE_TAG;
import static org.apache.asterix.om.types.ATypeTag.VALUE_TYPE_MAPPING;
+import static org.apache.asterix.om.util.container.ObjectFactories.BIT_SET_FACTORY;
+import static org.apache.asterix.om.util.container.ObjectFactories.STORAGE_FACTORY;
+import static org.apache.asterix.om.util.container.ObjectFactories.VOID_FACTORY;
import java.io.IOException;
import java.util.BitSet;
import java.util.List;
-import org.apache.asterix.builders.AbvsBuilderFactory;
import org.apache.asterix.dataflow.data.common.ILogicalBinaryComparator;
import org.apache.asterix.dataflow.data.common.ListAccessorUtil;
+import org.apache.asterix.dataflow.data.nontagged.CompareHashUtil;
import org.apache.asterix.formats.nontagged.BinaryComparatorFactoryProvider;
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.pointables.ARecordVisitablePointable;
import org.apache.asterix.om.pointables.PointableAllocator;
-import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
import org.apache.asterix.om.pointables.base.IVisitablePointable;
import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.ARecordType;
@@ -40,7 +42,6 @@ import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.AbstractCollectionType;
import org.apache.asterix.om.types.EnumDeserializer;
import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.om.util.container.IObjectFactory;
import org.apache.asterix.om.util.container.IObjectPool;
import org.apache.asterix.om.util.container.ListObjectPool;
import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
@@ -48,36 +49,30 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IMutableValueStorage;
import org.apache.hyracks.data.std.api.IPointable;
import org.apache.hyracks.data.std.api.IValueReference;
-import org.apache.hyracks.data.std.primitive.VoidPointable;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.util.string.UTF8StringUtil;
public class LogicalComplexBinaryComparator implements ILogicalBinaryComparator {
- private static final IObjectFactory<BitSet, Void> BIT_SET_FACTORY = (type) -> new BitSet();
- private static final IObjectFactory<IPointable, Void> VOID_FACTORY = (type) -> new VoidPointable();
private final IAType leftType;
private final IAType rightType;
private final boolean isEquality;
private final LogicalScalarBinaryComparator scalarComparator;
- private final IObjectPool<IMutableValueStorage, ATypeTag> storageAllocator;
+ private final IObjectPool<IMutableValueStorage, Void> storageAllocator;
private final IObjectPool<IPointable, Void> voidPointableAllocator;
private final IObjectPool<BitSet, Void> bitSetAllocator;
private final PointableAllocator pointableAllocator;
private final IBinaryComparator utf8Comp;
- private final StringBuilder builder;
- public LogicalComplexBinaryComparator(IAType leftType, IAType rightType, boolean isEquality) {
+ LogicalComplexBinaryComparator(IAType leftType, IAType rightType, boolean isEquality) {
this.leftType = leftType;
this.rightType = rightType;
this.isEquality = isEquality;
this.scalarComparator = new LogicalScalarBinaryComparator(isEquality);
- storageAllocator = new ListObjectPool<>(new AbvsBuilderFactory());
+ storageAllocator = new ListObjectPool<>(STORAGE_FACTORY);
voidPointableAllocator = new ListObjectPool<>(VOID_FACTORY);
bitSetAllocator = new ListObjectPool<>(BIT_SET_FACTORY);
pointableAllocator = new PointableAllocator();
utf8Comp = BinaryComparatorFactoryProvider.UTF8STRING_POINTABLE_INSTANCE.createBinaryComparator();
- builder = new StringBuilder();
}
@Override
@@ -91,15 +86,10 @@ public class LogicalComplexBinaryComparator implements ILogicalBinaryComparator
}
// make sure both left and right are complex types
if (!leftRuntimeTag.isDerivedType() || !rightRuntimeTag.isDerivedType()) {
- throw new IllegalStateException("Input types are not complex type");
- }
- try {
- return compareComplex(leftType, leftRuntimeTag, leftBytes, leftStart, leftLen, rightType, rightRuntimeTag,
- rightBytes, rightStart, rightLen);
- } finally {
- storageAllocator.reset();
- voidPointableAllocator.reset();
+ throw new IllegalStateException("Input data is not complex type");
}
+ return compareComplex(leftType, leftRuntimeTag, leftBytes, leftStart, leftLen, rightType, rightRuntimeTag,
+ rightBytes, rightStart, rightLen);
}
@Override
@@ -148,14 +138,8 @@ public class LogicalComplexBinaryComparator implements ILogicalBinaryComparator
if (leftRuntimeTag != rightRuntimeTag) {
return Result.INCOMPARABLE;
}
- IAType leftCompileType = TypeComputeUtils.getActualType(leftType);
- if (leftCompileType.getTypeTag() == ATypeTag.ANY) {
- leftCompileType = DefaultOpenFieldType.getDefaultOpenFieldType(leftRuntimeTag);
- }
- IAType rightCompileType = TypeComputeUtils.getActualType(rightType);
- if (rightCompileType.getTypeTag() == ATypeTag.ANY) {
- rightCompileType = DefaultOpenFieldType.getDefaultOpenFieldType(rightRuntimeTag);
- }
+ IAType leftCompileType = TypeComputeUtils.getActualTypeOrOpen(leftType, leftRuntimeTag);
+ IAType rightCompileType = TypeComputeUtils.getActualTypeOrOpen(rightType, rightRuntimeTag);
switch (leftRuntimeTag) {
case MULTISET:
return compareMultisets(leftCompileType, leftRuntimeTag, leftBytes, leftStart, rightCompileType,
@@ -187,13 +171,10 @@ public class LogicalComplexBinaryComparator implements ILogicalBinaryComparator
// TODO(ali): optimize to not need this storage, will require optimizing records comparison to not use visitable
ArrayBackedValueStorage leftStorage = (ArrayBackedValueStorage) storageAllocator.allocate(null);
ArrayBackedValueStorage rightStorage = (ArrayBackedValueStorage) storageAllocator.allocate(null);
- Result unknownResult = null;
Result determiningResult = null;
Result tempResult;
- byte leftItemTagByte;
- byte rightItemTagByte;
- ATypeTag leftItemRuntimeTag;
- ATypeTag rightItemRuntimeTag;
+ byte leftItemTagByte, rightItemTagByte;
+ ATypeTag leftItemRuntimeTag, rightItemRuntimeTag;
try {
for (int i = 0; i < leftNumItems && i < rightNumItems; i++) {
ListAccessorUtil.getItem(leftBytes, leftStart, i, leftListTag, leftItemTag, leftItem, leftStorage);
@@ -215,39 +196,19 @@ public class LogicalComplexBinaryComparator implements ILogicalBinaryComparator
rightItem.getLength());
}
- if (tempResult == Result.INCOMPARABLE) {
- return tempResult;
+ if (tempResult == Result.INCOMPARABLE || tempResult == Result.MISSING || tempResult == Result.NULL) {
+ return Result.INCOMPARABLE;
}
// skip to next pair if current one is equal or the result of the comparison has already been decided
- if (tempResult != Result.EQ && determiningResult == null) {
- // tempResult = NULL, MISSING, LT, GT
- if ((tempResult == Result.NULL || tempResult == Result.MISSING)) {
- // keep unknown response if there is no yet a determining result switching to missing if found
- if (unknownResult != Result.MISSING) {
- unknownResult = tempResult;
- }
- } else {
- // tempResult = LT, GT
- determiningResult = tempResult;
- }
+ if (determiningResult == null && tempResult != Result.EQ) {
+ determiningResult = tempResult;
}
}
- // reaching here means the two arrays are comparable
- if (isEquality && leftNumItems != rightNumItems) {
- return ILogicalBinaryComparator.asResult(Integer.compare(leftNumItems, rightNumItems));
- }
- // for >, < make unknownResult the determiningResult if unknownResult was encountered before finding one
- if (!isEquality && unknownResult != null) {
- determiningResult = unknownResult;
- }
if (determiningResult != null) {
return determiningResult;
}
- if (unknownResult != null) {
- return unknownResult;
- }
return ILogicalBinaryComparator.asResult(Integer.compare(leftNumItems, rightNumItems));
} catch (IOException e) {
throw HyracksDataException.create(e);
@@ -288,20 +249,12 @@ public class LogicalComplexBinaryComparator implements ILogicalBinaryComparator
List<IVisitablePointable> leftFieldNames = leftRecord.getFieldNames();
List<IVisitablePointable> rightFieldValues = rightRecord.getFieldValues();
List<IVisitablePointable> rightFieldNames = rightRecord.getFieldNames();
- IVisitablePointable leftFieldValue;
- IVisitablePointable leftFieldName;
- IVisitablePointable rightFieldValue;
- IVisitablePointable rightFieldName;
+ IVisitablePointable leftFieldValue, leftFieldName, rightFieldValue, rightFieldName;
int leftNumFields = leftFieldNames.size();
int rightNumFields = rightFieldNames.size();
- IAType leftFieldType;
- IAType rightFieldType;
- ATypeTag leftFTag;
- ATypeTag rightFTag;
+ IAType leftFieldType, rightFieldType;
+ ATypeTag leftFTag, rightFTag;
Result tempCompResult;
- Result unknownResult = null;
- Result determiningResult = null;
- String complexFieldName;
boolean foundFieldInRight;
boolean notEqual = false;
notMatched.set(0, rightNumFields);
@@ -311,6 +264,7 @@ public class LogicalComplexBinaryComparator implements ILogicalBinaryComparator
// ignore if the field value is missing
if (leftFTag != ATypeTag.MISSING) {
+ // start looking for the field in the right record
foundFieldInRight = false;
leftFieldName = leftFieldNames.get(i);
for (int k = 0; k < rightNumFields; k++) {
@@ -326,9 +280,8 @@ public class LogicalComplexBinaryComparator implements ILogicalBinaryComparator
if (leftFTag == ATypeTag.NULL || rightFTag == ATypeTag.NULL) {
tempCompResult = Result.NULL;
} else if (leftFTag.isDerivedType() && rightFTag.isDerivedType()) {
- complexFieldName = getComplexFieldName(leftFieldName);
- leftFieldType = getComplexFieldType(leftRecordType, complexFieldName, leftFTag);
- rightFieldType = getComplexFieldType(rightRecordType, complexFieldName, rightFTag);
+ leftFieldType = CompareHashUtil.getType(leftRecordType, i, leftFTag);
+ rightFieldType = CompareHashUtil.getType(rightRecordType, k, rightFTag);
tempCompResult =
compareComplex(leftFieldType, leftFTag, leftFieldValue.getByteArray(),
leftFieldValue.getStartOffset(), leftFieldValue.getLength(),
@@ -341,15 +294,12 @@ public class LogicalComplexBinaryComparator implements ILogicalBinaryComparator
rightFieldValue.getLength());
}
- if (tempCompResult == Result.INCOMPARABLE) {
- return tempCompResult;
+ if (tempCompResult == Result.INCOMPARABLE || tempCompResult == Result.MISSING
+ || tempCompResult == Result.NULL) {
+ return Result.INCOMPARABLE;
}
- if (tempCompResult == Result.MISSING || tempCompResult == Result.NULL) {
- if (unknownResult != Result.MISSING) {
- unknownResult = tempCompResult;
- }
- } else if (tempCompResult != Result.EQ && determiningResult == null) {
- determiningResult = tempCompResult;
+ if (tempCompResult != Result.EQ) {
+ notEqual = true;
}
}
break;
@@ -365,27 +315,18 @@ public class LogicalComplexBinaryComparator implements ILogicalBinaryComparator
// LT or GT does not make a difference since this is an answer to equality
return Result.LT;
}
- // two fields with the same name but having different values
- if (determiningResult != null) {
- return determiningResult;
- }
// check if there is a field in the right record that does not exist in left record
byte rightFieldTag;
- for (int i = 0; i < rightNumFields; i++) {
+ for (int i = notMatched.nextSetBit(0); i >= 0 && i < rightNumFields; i = notMatched.nextSetBit(i + 1)) {
rightFieldValue = rightFieldValues.get(i);
rightFieldTag = rightFieldValue.getByteArray()[rightFieldValue.getStartOffset()];
- if (notMatched.get(i) && rightFieldTag != SERIALIZED_MISSING_TYPE_TAG) {
- notEqual = true;
- break;
+ if (rightFieldTag != SERIALIZED_MISSING_TYPE_TAG) {
+ // LT or GT does not make a difference since this is an answer to equality
+ return Result.LT;
}
}
- if (notEqual) {
- return Result.LT;
- }
+
// reaching here means every field in the left record exists in the right and vice versa
- if (unknownResult != null) {
- return unknownResult;
- }
return Result.EQ;
} finally {
pointableAllocator.freeRecord(rightRecord);
@@ -394,16 +335,6 @@ public class LogicalComplexBinaryComparator implements ILogicalBinaryComparator
}
}
- private IAType getComplexFieldType(ARecordType recordType, String fieldName, ATypeTag fieldRuntimeTag) {
- IAType fieldType = recordType.getFieldType(fieldName);
- return fieldType == null ? DefaultOpenFieldType.getDefaultOpenFieldType(fieldRuntimeTag) : fieldType;
- }
-
- private String getComplexFieldName(IValueReference fieldName) {
- builder.setLength(0);
- return UTF8StringUtil.toString(builder, fieldName.getByteArray(), fieldName.getStartOffset() + 1).toString();
- }
-
private boolean equalNames(IValueReference fieldName1, IValueReference fieldName2) throws HyracksDataException {
// TODO(ali): refactor with PointableHelper and move it from runtime package
return utf8Comp.compare(fieldName1.getByteArray(), fieldName1.getStartOffset() + 1, fieldName1.getLength() - 1,
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/LogicalScalarBinaryComparator.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/LogicalScalarBinaryComparator.java
index f4896dc..a25c90e 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/LogicalScalarBinaryComparator.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/comparators/LogicalScalarBinaryComparator.java
@@ -74,7 +74,7 @@ public class LogicalScalarBinaryComparator implements ILogicalBinaryComparator {
private final boolean isEquality;
- public LogicalScalarBinaryComparator(boolean isEquality) {
+ LogicalScalarBinaryComparator(boolean isEquality) {
this.isEquality = isEquality;
}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/hash/AMurmurHash3BinaryHashFunctionFamily.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/hash/AMurmurHash3BinaryHashFunctionFamily.java
index 7cd69ab..cc33faa 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/hash/AMurmurHash3BinaryHashFunctionFamily.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/hash/AMurmurHash3BinaryHashFunctionFamily.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.dataflow.data.nontagged.hash;
+import static org.apache.asterix.om.types.ATypeTag.VALUE_TYPE_MAPPING;
+
import java.io.DataOutput;
import java.io.IOException;
import java.util.Comparator;
@@ -82,7 +84,7 @@ public class AMurmurHash3BinaryHashFunctionFamily implements IBinaryHashFunction
private final ArrayBackedValueStorage valueBuffer = new ArrayBackedValueStorage();
private final DataOutput valueOut = valueBuffer.getDataOutput();
private final IObjectPool<IPointable, Void> voidPointableAllocator;
- private final IObjectPool<IMutableValueStorage, ATypeTag> storageAllocator;
+ private final IObjectPool<IMutableValueStorage, Void> storageAllocator;
private final IAType type;
private final int seed;
// used for record hashing, sorting field names first
@@ -189,6 +191,7 @@ public class AMurmurHash3BinaryHashFunctionFamily implements IBinaryHashFunction
CompareHashUtil.addToHeap(fieldsNames, fieldsValues, namesHeap);
IVisitablePointable fieldName, fieldValue;
IAType fieldType;
+ ATypeTag fieldTag;
int hash = 0;
int fieldIdx;
while (!namesHeap.isEmpty()) {
@@ -196,7 +199,8 @@ public class AMurmurHash3BinaryHashFunctionFamily implements IBinaryHashFunction
// TODO(ali): currently doing another lookup to find the target field index and get its value & type
fieldIdx = CompareHashUtil.getIndex(fieldsNames, fieldName);
fieldValue = fieldsValues.get(fieldIdx);
- fieldType = CompareHashUtil.getType(recordType, fieldIdx, fieldValue);
+ fieldTag = VALUE_TYPE_MAPPING[fieldValue.getByteArray()[fieldValue.getStartOffset()]];
+ fieldType = CompareHashUtil.getType(recordType, fieldIdx, fieldTag);
hash ^= MurmurHash3BinaryHash.hash(fieldName.getByteArray(), fieldName.getStartOffset(),
fieldName.getLength(), seed)
^ hash(fieldType, fieldValue.getByteArray(), fieldValue.getStartOffset(),
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/container/ObjectFactories.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/container/ObjectFactories.java
index e9cb345..763e1d4 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/container/ObjectFactories.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/container/ObjectFactories.java
@@ -18,19 +18,26 @@
*/
package org.apache.asterix.om.util.container;
-import org.apache.asterix.builders.AbvsBuilderFactory;
-import org.apache.asterix.om.types.ATypeTag;
+import java.util.BitSet;
+
import org.apache.hyracks.data.std.api.IMutableValueStorage;
import org.apache.hyracks.data.std.api.IPointable;
import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
// TODO(ali): look for all classes creating factories and extract them to here
+
+/**
+ * Object factories must be used in conjunction with {@link IObjectPool} to reuse objects. They should not be used
+ * to create objects outside the context of a pool.
+ */
public class ObjectFactories {
private ObjectFactories() {
}
public static final IObjectFactory<IPointable, Void> VOID_FACTORY = (type) -> new VoidPointable();
- // TODO(ali): use lambda for the storage, too
- public static final IObjectFactory<IMutableValueStorage, ATypeTag> STORAGE_FACTORY = new AbvsBuilderFactory();
+ public static final IObjectFactory<IMutableValueStorage, Void> STORAGE_FACTORY =
+ (type) -> new ArrayBackedValueStorage();
+ public static final IObjectFactory<BitSet, Void> BIT_SET_FACTORY = (type) -> new BitSet();
}