You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ti...@apache.org on 2016/01/11 17:20:47 UTC
[2/6] incubator-asterixdb git commit: ASTERIXDB-1187,
ASTERIXDB-1162 and ASTERIXDB-1252 fixes, plus new internal functions
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordAddFieldsTypeComputer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordAddFieldsTypeComputer.java b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordAddFieldsTypeComputer.java
new file mode 100644
index 0000000..4f8d1f2
--- /dev/null
+++ b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordAddFieldsTypeComputer.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.om.typecomputer.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.constants.AsterixConstantValue;
+import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.TypeHelper;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+public class RecordAddFieldsTypeComputer implements IResultTypeComputer {
+ public static final RecordAddFieldsTypeComputer INSTANCE = new RecordAddFieldsTypeComputer();
+
+ private static final String FIELD_NAME_NAME = "field-name";
+ private static final String FIELD_VALUE_VALUE = "field-value";
+
+ private RecordAddFieldsTypeComputer() {
+ }
+
+ @Override
+ public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env,
+ IMetadataProvider<?, ?> metadataProvider) throws AlgebricksException {
+
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expression;
+ IAType type0 = (IAType) env.getType(funcExpr.getArguments().get(0).getValue());
+
+ ARecordType inputRecordType = TypeComputerUtils.extractRecordType(type0);
+ if (inputRecordType == null) {
+ throw new AlgebricksException("Input record cannot be null");
+ }
+
+ AbstractLogicalExpression arg1 = (AbstractLogicalExpression) funcExpr.getArguments().get(1).getValue();
+ IAType type1 = (IAType) env.getType(arg1);
+ AOrderedListType inputOrderedListType = TypeComputerUtils.extractOrderedListType(type1);
+ if (inputOrderedListType == null) {
+ return inputRecordType;
+ }
+
+ boolean nullable = TypeHelper.canBeNull(type0) || TypeHelper.canBeNull(type1);
+ Map<String, IAType> additionalFields = new HashMap<>();
+ List<String> resultFieldNames = new ArrayList<>();
+ List<IAType> resultFieldTypes = new ArrayList<>();
+
+ resultFieldNames.addAll(Arrays.asList(inputRecordType.getFieldNames()));
+ Collections.sort(resultFieldNames);
+
+ for (String fieldName : resultFieldNames) {
+ try {
+ if (inputRecordType.getFieldType(fieldName).getTypeTag() == ATypeTag.RECORD) {
+ ARecordType nestedType = (ARecordType) inputRecordType.getFieldType(fieldName);
+ //Deep Copy prevents altering of input types
+ resultFieldTypes.add(nestedType.deepCopy(nestedType));
+ } else {
+ resultFieldTypes.add(inputRecordType.getFieldType(fieldName));
+ }
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ if (!containsVariable(arg1)) {
+ AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) arg1;
+ List<Mutable<ILogicalExpression>> args = f.getArguments();
+
+ AString fieldName = null;
+ IAType fieldType = null;
+
+ // Iterating through the orderlist input
+ for (Mutable<ILogicalExpression> arg : args) {
+ AbstractFunctionCallExpression recConsExpr = (AbstractFunctionCallExpression) arg.getValue();
+ ARecordType rtype = TypeComputerUtils.extractRecordType((IAType) env.getType(recConsExpr));
+ if (rtype != null) {
+ String[] fn = rtype.getFieldNames();
+ IAType[] ft = rtype.getFieldTypes();
+ for (int j = 0; j < fn.length; j++) {
+ if (fn[j].equals(FIELD_NAME_NAME)) {
+ ILogicalExpression fieldNameExpr = recConsExpr.getArguments().get(j).getValue();
+ switch (fieldNameExpr.getExpressionTag()) {
+ case CONSTANT: // Top fields only
+ IAObject object = ((AsterixConstantValue) ((ConstantExpression) fieldNameExpr)
+ .getValue()).getObject();
+ if (object.getType().getTypeTag() == ATypeTag.STRING) {
+ // Get the actual "field-name" string
+ ILogicalExpression recFieldExpr = recConsExpr.getArguments().get(j + 1)
+ .getValue();
+ if (recFieldExpr.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+ fieldName = (AString) ((AsterixConstantValue) ((ConstantExpression) recFieldExpr)
+ .getValue()).getObject();
+ }
+ }
+ break;
+ default:
+ throw new AlgebricksException(fieldNameExpr + " is not supported.");
+ }
+ } else if (fn[j].equals(FIELD_VALUE_VALUE)) {
+ fieldType = ft[j];
+ }
+ }
+ if (fieldName != null) {
+ additionalFields.put(fieldName.getStringValue(), fieldType);
+ }
+ }
+ }
+
+ if (!additionalFields.isEmpty()) {
+ Iterator<Map.Entry<String, IAType>> it = additionalFields.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry<String, IAType> entry = it.next();
+ resultFieldNames.add(entry.getKey());
+ resultFieldTypes.add(entry.getValue());
+ }
+ }
+ } // If variable ignore, deal with the addition at runtime
+
+ String resultTypeName = "appended(" + inputRecordType.getTypeName() + ")";
+ int n = resultFieldNames.size();
+ IAType resultType = new ARecordType(resultTypeName, resultFieldNames.toArray(new String[n]),
+ resultFieldTypes.toArray(new IAType[n]), true);
+ if (nullable) {
+ resultType = AUnionType.createNullableType(resultType);
+ }
+ return resultType;
+ }
+
+ // Handle variable as input
+ private boolean containsVariable(ILogicalExpression expression) {
+ if (expression.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expression;
+ List<Mutable<ILogicalExpression>> args = f.getArguments();
+ for (Mutable<ILogicalExpression> arg : args) {
+ ILogicalExpression subExpression = arg.getValue();
+ switch (subExpression.getExpressionTag()) {
+ case VARIABLE:
+ return true;
+ case CONSTANT:
+ return false;
+ default: //FUNCTION_CALL
+ return containsVariable(subExpression);
+ }
+ }
+ }
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordMergeTypeComputer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordMergeTypeComputer.java b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordMergeTypeComputer.java
index 83f4f44..653b26e 100644
--- a/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordMergeTypeComputer.java
+++ b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordMergeTypeComputer.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
import org.apache.asterix.om.types.ARecordType;
@@ -44,21 +43,6 @@ public class RecordMergeTypeComputer implements IResultTypeComputer {
private RecordMergeTypeComputer() {
}
- public static ARecordType extractRecordType(IAType t) {
- if (t.getTypeTag() == ATypeTag.RECORD) {
- return (ARecordType) t;
- }
-
- if (t.getTypeTag() == ATypeTag.UNION) {
- IAType innerType = ((AUnionType) t).getNullableType();
- if (innerType.getTypeTag() == ATypeTag.RECORD) {
- return (ARecordType) innerType;
- }
- }
-
- return null;
- }
-
@Override
public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env,
IMetadataProvider<?, ?> metadataProvider) throws AlgebricksException {
@@ -66,12 +50,12 @@ public class RecordMergeTypeComputer implements IResultTypeComputer {
IAType t0 = (IAType) env.getType(f.getArguments().get(0).getValue());
IAType t1 = (IAType) env.getType(f.getArguments().get(1).getValue());
boolean nullable = TypeHelper.canBeNull(t0) || TypeHelper.canBeNull(t1);
- ARecordType recType0 = extractRecordType(t0);
- ARecordType recType1 = extractRecordType(t1);
+ ARecordType recType0 = TypeComputerUtils.extractRecordType(t0);
+ ARecordType recType1 = TypeComputerUtils.extractRecordType(t1);
if (recType0 == null || recType1 == null) {
- throw new AlgebricksException(
- "record-merge expects possibly NULL records as arguments, but got (" + t0 + ", " + t1 + ")");
+ throw new AlgebricksException("record-merge expects possibly NULL records as arguments, but got (" + t0
+ + ", " + t1 + ")");
}
List<String> resultFieldNames = new ArrayList<>();
@@ -79,6 +63,7 @@ public class RecordMergeTypeComputer implements IResultTypeComputer {
resultFieldNames.add(fieldName);
}
Collections.sort(resultFieldNames);
+
List<IAType> resultFieldTypes = new ArrayList<>();
for (String fieldName : resultFieldNames) {
try {
@@ -96,20 +81,27 @@ public class RecordMergeTypeComputer implements IResultTypeComputer {
List<String> additionalFieldNames = new ArrayList<>();
List<IAType> additionalFieldTypes = new ArrayList<>();
- for (int i = 0; i < recType1.getFieldNames().length; ++i) {
- String fieldName = recType1.getFieldNames()[i];
- IAType fieldType = recType1.getFieldTypes()[i];
- int pos = Collections.binarySearch(resultFieldNames, fieldName);
+ String fieldNames[] = recType1.getFieldNames();
+ IAType fieldTypes[] = recType1.getFieldTypes();
+ for (int i = 0; i < fieldNames.length; ++i) {
+ int pos = Collections.binarySearch(resultFieldNames, fieldNames[i]);
if (pos >= 0) {
+ IAType resultFieldType = resultFieldTypes.get(pos);
+ if (resultFieldType.getTypeTag() != fieldTypes[i].getTypeTag()) {
+ throw new AlgebricksException("Duplicate field " + fieldNames[i] + " encountered");
+ }
try {
- resultFieldTypes.set(pos, mergedNestedType(fieldType, resultFieldTypes.get(pos)));
+ // Assuming fieldTypes[i].getTypeTag() = resultFieldType.getTypeTag()
+ if (fieldTypes[i].getTypeTag() == ATypeTag.RECORD) {
+ resultFieldTypes.set(pos, mergedNestedType(fieldTypes[i], resultFieldType));
+ }
} catch (AsterixException e) {
throw new AlgebricksException(e);
}
} else {
- additionalFieldNames.add(fieldName);
- additionalFieldTypes.add(fieldType);
+ additionalFieldNames.add(fieldNames[i]);
+ additionalFieldTypes.add(fieldTypes[i]);
}
}
@@ -117,9 +109,9 @@ public class RecordMergeTypeComputer implements IResultTypeComputer {
resultFieldTypes.addAll(additionalFieldTypes);
String resultTypeName = "merged(" + recType0.getTypeName() + ", " + recType1.getTypeName() + ")";
boolean isOpen = recType0.isOpen() || recType1.isOpen();
- IAType resultType = null;
- resultType = new ARecordType(resultTypeName, resultFieldNames.toArray(new String[] {}),
- resultFieldTypes.toArray(new IAType[] {}), isOpen);
+
+ IAType resultType = new ARecordType(resultTypeName, resultFieldNames.toArray(new String[] {}),
+ resultFieldTypes.toArray(new IAType[] {}), isOpen);
if (nullable) {
resultType = AUnionType.createNullableType(resultType);
@@ -127,32 +119,32 @@ public class RecordMergeTypeComputer implements IResultTypeComputer {
return resultType;
}
- IAType mergedNestedType(IAType fieldType1, IAType fieldType0) throws AlgebricksException, AsterixException {
+ private IAType mergedNestedType(IAType fieldType1, IAType fieldType0) throws AlgebricksException, AsterixException {
if (fieldType1.getTypeTag() != ATypeTag.RECORD || fieldType0.getTypeTag() != ATypeTag.RECORD) {
- throw new AlgebricksException("Duplicate field \"" + fieldType1.getTypeName() + "\" encountered");
+ throw new AlgebricksException("Duplicate field " + fieldType1.getTypeName() + " encountered");
}
- ARecordType returnType = (ARecordType) fieldType0;
+ ARecordType resultType = (ARecordType) fieldType0;
ARecordType fieldType1Copy = (ARecordType) fieldType1;
for (int i = 0; i < fieldType1Copy.getFieldTypes().length; i++) {
try {
- int pos = returnType.getFieldIndex(fieldType1Copy.getFieldNames()[i]);
+ int pos = resultType.getFieldIndex(fieldType1Copy.getFieldNames()[i]);
if (pos >= 0) {
- if (fieldType1Copy.getFieldTypes()[i].getTypeTag() != ATypeTag.RECORD) {
- break;
+ // If a sub-record do merge, else ignore and let the values decide what to do
+ if (fieldType1Copy.getFieldTypes()[i].getTypeTag() == ATypeTag.RECORD) {
+ IAType[] oldTypes = resultType.getFieldTypes();
+ oldTypes[pos] = mergedNestedType(fieldType1Copy.getFieldTypes()[i],
+ resultType.getFieldTypes()[pos]);
+ resultType = new ARecordType(resultType.getTypeName(), resultType.getFieldNames(), oldTypes,
+ resultType.isOpen());
}
- IAType[] oldTypes = returnType.getFieldTypes();
- oldTypes[pos] = mergedNestedType(fieldType1Copy.getFieldTypes()[i],
- returnType.getFieldTypes()[pos]);
- returnType = new ARecordType(returnType.getTypeName(), returnType.getFieldNames(), oldTypes,
- returnType.isOpen());
} else {
- IAType[] combinedFieldTypes = ArrayUtils.addAll(returnType.getFieldTypes().clone(),
+ IAType[] combinedFieldTypes = ArrayUtils.addAll(resultType.getFieldTypes().clone(),
fieldType1Copy.getFieldTypes()[i]);
- returnType = new ARecordType(returnType.getTypeName(),
- ArrayUtils.addAll(returnType.getFieldNames(), fieldType1Copy.getFieldNames()[i]),
- combinedFieldTypes, returnType.isOpen());
+ resultType = new ARecordType(resultType.getTypeName(), ArrayUtils.addAll(
+ resultType.getFieldNames(), fieldType1Copy.getFieldNames()[i]), combinedFieldTypes,
+ resultType.isOpen());
}
} catch (IOException | AsterixException e) {
@@ -160,6 +152,6 @@ public class RecordMergeTypeComputer implements IResultTypeComputer {
}
}
- return returnType;
+ return resultType;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRemoveFieldsTypeComputer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRemoveFieldsTypeComputer.java b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRemoveFieldsTypeComputer.java
new file mode 100644
index 0000000..9f4d155
--- /dev/null
+++ b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRemoveFieldsTypeComputer.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.om.typecomputer.impl;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.asterix.om.base.AOrderedList;
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.constants.AsterixConstantValue;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+/**
+ * Cases to support:
+ * remove-fields($record, ["foo", ["bar", "access"]]),
+ * where ["bar", "access"] is equivalent to the path bar->access
+ */
+public class RecordRemoveFieldsTypeComputer implements IResultTypeComputer {
+
+ public static final RecordRemoveFieldsTypeComputer INSTANCE = new RecordRemoveFieldsTypeComputer();
+
+ private RecordRemoveFieldsTypeComputer() {
+ }
+
+ private void getPathFromConstantExpression(ILogicalExpression expression, Set<String> fieldNameSet,
+ List<List<String>> pathList) throws AlgebricksException {
+ ConstantExpression ce = (ConstantExpression) expression;
+ if (!(ce.getValue() instanceof AsterixConstantValue)) {
+ throw new AlgebricksException("Expecting a list of strings and found " + ce.getValue() + " instead.");
+ }
+ IAObject item = ((AsterixConstantValue) ce.getValue()).getObject();
+ ATypeTag type = item.getType().getTypeTag();
+
+ switch (type) {
+ case STRING:
+ String fn = ((AString) item).getStringValue();
+ fieldNameSet.add(fn);
+ break;
+ case ORDEREDLIST:
+ AOrderedList pathOrdereList = (AOrderedList) item;
+ String fieldName = ((AString) pathOrdereList.getItem(0)).getStringValue();
+ fieldNameSet.add(fieldName);
+ List<String> path = new ArrayList<>();
+ for (int i = 0; i < pathOrdereList.size(); i++) {
+ path.add(((AString) pathOrdereList.getItem(i)).getStringValue());
+ }
+ pathList.add(path);
+ break;
+ default:
+ throw new AlgebricksException("Unsupport type: " + type);
+ }
+ }
+
+ private List<String> getListFromExpression(ILogicalExpression expression) throws AlgebricksException {
+ AbstractFunctionCallExpression funcExp = (AbstractFunctionCallExpression) expression;
+ List<Mutable<ILogicalExpression>> args = funcExp.getArguments();
+
+ List<String> list = new ArrayList<>();
+ for (Mutable<ILogicalExpression> arg : args) {
+ // At this point all elements has to be a constant
+ // Input list has only one level of nesting (list of list or list of strings)
+ ConstantExpression ce = (ConstantExpression) arg.getValue();
+ if (!(ce.getValue() instanceof AsterixConstantValue)) {
+ throw new AlgebricksException("Expecting a list of strings and found " + ce.getValue() + " instead.");
+ }
+ IAObject item = ((AsterixConstantValue) ce.getValue()).getObject();
+ ATypeTag type = item.getType().getTypeTag();
+ if (type == ATypeTag.STRING) {
+ list.add(((AString) item).getStringValue());
+ } else {
+ throw new AlgebricksException(type + " is currently not supported. Please check your function call.");
+ }
+ }
+
+ return list;
+ }
+
+ private void getPathFromFunctionExpression(ILogicalExpression expression, Set<String> fieldNameSet,
+ List<List<String>> pathList) throws AlgebricksException {
+
+ List<String> path = getListFromExpression(expression);
+ // Add the path head to remove set
+ fieldNameSet.add(path.get(0));
+ pathList.add(path);
+
+ }
+
+ private void computeTypeFromNonConstantExpression(ILogicalExpression expression, Set<String> fieldNameSet,
+ List<List<String>> pathList) throws AlgebricksException {
+ AbstractFunctionCallExpression funcExp = (AbstractFunctionCallExpression) expression;
+ List<Mutable<ILogicalExpression>> args = funcExp.getArguments();
+
+ for (Mutable<ILogicalExpression> arg : args) {
+ ILogicalExpression le = arg.getValue();
+ switch (le.getExpressionTag()) {
+ case CONSTANT:
+ getPathFromConstantExpression(le, fieldNameSet, pathList);
+ break;
+ case FUNCTION_CALL:
+ getPathFromFunctionExpression(le, fieldNameSet, pathList);
+ break;
+ default:
+ throw new AlgebricksException("Unsupported expression: " + le);
+ }
+ }
+ }
+
+ @Override
+ public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env,
+ IMetadataProvider<?, ?> metadataProvider) throws AlgebricksException {
+
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expression;
+ IAType type0 = (IAType) env.getType(funcExpr.getArguments().get(0).getValue());
+
+ List<List<String>> pathList = new ArrayList<>();
+ Set<String> fieldNameSet = new HashSet<>();
+ Deque<String> fieldPathStack = new ArrayDeque<>();
+
+ ARecordType inputRecordType = NonTaggedFieldAccessByNameResultType.getRecordTypeFromType(type0, expression);
+ if (inputRecordType == null) {
+ return BuiltinType.ANY;
+ }
+
+ AbstractLogicalExpression arg1 = (AbstractLogicalExpression) funcExpr.getArguments().get(1).getValue();
+ IAType inputListType = (IAType) env.getType(arg1);
+ AOrderedListType inputOrderedListType = TypeComputerUtils.extractOrderedListType(inputListType);
+ if (inputOrderedListType == null) {
+ throw new AlgebricksException(
+ "The function 'remove-fields' expects an ordered list as the second argument, but got "
+ + inputListType);
+ }
+
+ ATypeTag tt = inputOrderedListType.getItemType().getTypeTag();
+ if (tt == ATypeTag.STRING) { // If top-fieldlist
+ if (setFieldNameSet(arg1, fieldNameSet)) {
+ return buildOutputType(fieldPathStack, inputRecordType, fieldNameSet, pathList);
+ } else {
+ return DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+ }
+ } else { // tt == ATypeTag.ANY, meaning the list is nested
+ computeTypeFromNonConstantExpression(arg1, fieldNameSet, pathList);
+ IAType resultType = buildOutputType(fieldPathStack, inputRecordType, fieldNameSet, pathList);
+ return resultType;
+ }
+ }
+
+ private boolean setFieldNameSet(ILogicalExpression expr, Set<String> fieldNameSet) {
+ if (expr.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+ AOrderedList orderedList = (AOrderedList) (((AsterixConstantValue) ((ConstantExpression) expr).getValue())
+ .getObject());
+ for (int i = 0; i < orderedList.size(); i++) {
+ AString as = (AString) orderedList.getItem(i);
+ fieldNameSet.add(as.getStringValue());
+ }
+ return true; // Success
+ }
+ return false;
+ }
+
+ private void addField(ARecordType inputRecordType, String fieldName, List<String> resultFieldNames, List<IAType>
+ resultFieldTypes)
+ throws AlgebricksException {
+ try {
+ resultFieldNames.add(fieldName);
+ if (inputRecordType.getFieldType(fieldName).getTypeTag() == ATypeTag.RECORD) {
+ ARecordType nestedType = (ARecordType) inputRecordType.getFieldType(fieldName);
+ //Deep Copy prevents altering of input types
+ resultFieldTypes.add(nestedType.deepCopy(nestedType));
+ } else {
+ resultFieldTypes.add(inputRecordType.getFieldType(fieldName));
+ }
+
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ private IAType buildOutputType(Deque<String> fieldPathStack, ARecordType inputRecordType, Set<String> fieldNameSet,
+ List<List<String>> pathList) throws AlgebricksException {
+ IAType resultType;
+ List<String> resultFieldNames = new ArrayList<>();
+ List<IAType> resultFieldTypes = new ArrayList<>();
+
+ String[] fieldNames = inputRecordType.getFieldNames();
+ IAType[] fieldTypes = inputRecordType.getFieldTypes();
+
+ for (int i = 0; i < fieldNames.length; i++) {
+ if (!fieldNameSet.contains(fieldNames[i])) { // The main field is to be kept
+ addField(inputRecordType, fieldNames[i], resultFieldNames, resultFieldTypes);
+ } else if (!pathList.isEmpty()) { // Further check needed for nested fields
+ if (fieldTypes[i].getTypeTag() == ATypeTag.RECORD) {
+ ARecordType subRecord = (ARecordType) fieldTypes[i];
+
+ fieldPathStack.push(fieldNames[i]);
+ subRecord = deepCheckAndCopy(fieldPathStack, subRecord, pathList, inputRecordType.isOpen());
+ fieldPathStack.pop();
+ if (subRecord != null) {
+ resultFieldNames.add(fieldNames[i]);
+ resultFieldTypes.add(subRecord);
+ }
+ }
+ }
+ }
+
+ int n = resultFieldNames.size();
+ String resultTypeName = "result-record(" + inputRecordType.getTypeName() + ")";
+
+ return new ARecordType(resultTypeName, resultFieldNames.toArray(new String[n]),
+ resultFieldTypes.toArray(new IAType[n]), true); // Make the output type open always
+
+ }
+
+ /**
+ * Comparison elements of two paths
+ * Note: l2 uses a LIFO insert and removal.
+ */
+ private <E> boolean isEqualPaths(List<E> l1, Deque<E> l2) {
+ if ((l1 == null) || (l2 == null))
+ return false;
+
+ if (l1.size() != l2.size())
+ return false;
+
+ Iterator<E> it2 = l2.iterator();
+
+ int len = l1.size();
+ for (int i = len - 1; i >= 0; i--) {
+ E o1 = l1.get(i);
+ E o2 = it2.next();
+ if (!o1.equals(o2))
+ return false;
+ }
+ return true;
+ }
+
+ private boolean isRemovePath(Deque<String> fieldPath, List<List<String>> pathList) {
+ for (List<String> removePath : pathList) {
+ if (isEqualPaths(removePath, fieldPath)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /*
+ A method to deep copy a record the path validation
+ i.e., keep only fields that are valid
+ */
+ private ARecordType deepCheckAndCopy(Deque<String> fieldPath, ARecordType srcRecType, List<List<String>>
+ pathList, boolean isOpen)
+ throws AlgebricksException {
+ // Make sure the current path is valid before going further
+ if (isRemovePath(fieldPath, pathList)) {
+ return null;
+ }
+
+ String srcFieldNames[] = srcRecType.getFieldNames();
+ IAType srcFieldTypes[] = srcRecType.getFieldTypes();
+
+ List<IAType> destFieldTypes = new ArrayList<>();
+ List<String> destFieldNames = new ArrayList<>();
+
+ for (int i = 0; i < srcFieldNames.length; i++) {
+ fieldPath.push(srcFieldNames[i]);
+ if (!isRemovePath(fieldPath, pathList)) {
+ if (srcFieldTypes[i].getTypeTag() == ATypeTag.RECORD) {
+ ARecordType subRecord = (ARecordType) srcFieldTypes[i];
+ subRecord = deepCheckAndCopy(fieldPath, subRecord, pathList, isOpen);
+ if (subRecord != null) {
+ destFieldNames.add(srcFieldNames[i]);
+ destFieldTypes.add(subRecord);
+ }
+ } else {
+ destFieldNames.add(srcFieldNames[i]);
+ destFieldTypes.add(srcFieldTypes[i]);
+ }
+ }
+ fieldPath.pop();
+ }
+
+ int n = destFieldNames.size();
+ if (n == 0) {
+ return null;
+ }
+ return new ARecordType(srcRecType.getTypeName(), destFieldNames.toArray(new String[n]),
+ destFieldTypes.toArray(new IAType[n]), isOpen);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputerUtils.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputerUtils.java b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputerUtils.java
new file mode 100644
index 0000000..cba988b
--- /dev/null
+++ b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputerUtils.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.typecomputer.impl;
+
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.AUnorderedListType;
+import org.apache.asterix.om.types.IAType;
+
+public class TypeComputerUtils {
+
+ public static ARecordType extractRecordType(IAType t) {
+ if (t.getTypeTag() == ATypeTag.RECORD) {
+ return (ARecordType) t;
+ }
+
+ if (t.getTypeTag() == ATypeTag.UNION) {
+ IAType innerType = ((AUnionType) t).getUnionList().get(1);
+ if (innerType.getTypeTag() == ATypeTag.RECORD) {
+ return (ARecordType) innerType;
+ }
+ }
+
+ return null;
+ }
+
+ public static AOrderedListType extractOrderedListType(IAType t) {
+ if (t.getTypeTag() == ATypeTag.ORDEREDLIST) {
+ return (AOrderedListType) t;
+ }
+
+ if (t.getTypeTag() == ATypeTag.UNION) {
+ IAType innerType = ((AUnionType) t).getUnionList().get(1);
+ if (innerType.getTypeTag() == ATypeTag.ORDEREDLIST) {
+ return (AOrderedListType) innerType;
+ }
+ }
+
+ return null;
+ }
+
+ public static AUnorderedListType extractUnorderedListType(IAType t) {
+ if (t.getTypeTag() == ATypeTag.UNORDEREDLIST) {
+ return (AUnorderedListType) t;
+ }
+
+ if (t.getTypeTag() == ATypeTag.UNION) {
+ IAType innerType = ((AUnionType) t).getUnionList().get(1);
+ if (innerType.getTypeTag() == ATypeTag.UNORDEREDLIST) {
+ return (AUnorderedListType) innerType;
+ }
+ }
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/ATypeHierarchy.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/ATypeHierarchy.java b/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/ATypeHierarchy.java
index 6524930..0b6b45d 100644
--- a/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/ATypeHierarchy.java
+++ b/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/ATypeHierarchy.java
@@ -45,13 +45,6 @@ import org.apache.hyracks.data.std.primitive.ShortPointable;
public class ATypeHierarchy {
- public static enum Domain {
- SPATIAL,
- NUMERIC,
- LIST,
- ANY
- }
-
private static BitSet typePromotionHierachyMap = new BitSet(ATypeTag.TYPE_COUNT * ATypeTag.TYPE_COUNT);
private static BitSet typeDemotionHierachyMap = new BitSet(ATypeTag.TYPE_COUNT * ATypeTag.TYPE_COUNT);
private static HashMap<Integer, ITypeConvertComputer> promoteComputerMap = new HashMap<Integer, ITypeConvertComputer>();
@@ -125,6 +118,10 @@ public class ATypeHierarchy {
hierarchyDomains.put(ATypeTag.UNORDEREDLIST, Domain.LIST);
}
+ public static Domain getTypeDomain(ATypeTag tag) {
+ return hierarchyDomains.get(tag);
+ }
+
public static boolean isSameTypeDomain(ATypeTag tag1, ATypeTag tag2, boolean useListDomain) {
Domain tagHierarchy1 = hierarchyDomains.get(tag1);
Domain tagHierarchy2 = hierarchyDomains.get(tag2);
@@ -805,4 +802,11 @@ public class ATypeHierarchy {
return value;
}
+ public static enum Domain {
+ SPATIAL,
+ NUMERIC,
+ LIST,
+ ANY
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/comparisons/DeepEqualAssessor.java
----------------------------------------------------------------------
diff --git a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/comparisons/DeepEqualAssessor.java b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/comparisons/DeepEqualAssessor.java
new file mode 100644
index 0000000..b4e0375
--- /dev/null
+++ b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/comparisons/DeepEqualAssessor.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.comparisons;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy.Domain;
+import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
+import org.apache.asterix.runtime.evaluators.visitors.DeepEqualityVisitor;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Use {@link DeepEqualityVisitor} to assess the deep equality between two
+ * pointable values, including oredered and unordered lists, record values, etc.
+ * Example: Let IVisitablePointable leftPointable, IVisitablePointable rightPointable be two
+ * value references. To assess their equality, simply use
+ * DeepEqualAssessor dea = new DeepEqualAssessor();
+ * boolean isEqual = dea.isEqual(leftPointable, rightPointable);
+ */
+
+public class DeepEqualAssessor {
+ private final DeepEqualityVisitor equalityVisitor = new DeepEqualityVisitor();
+
+ public boolean isEqual(IVisitablePointable leftPointable, IVisitablePointable rightPointable)
+ throws AlgebricksException, AsterixException {
+
+ if (leftPointable == null || rightPointable == null) {
+ return false;
+ }
+
+ if (leftPointable.equals(rightPointable)) {
+ return true;
+ }
+
+ ATypeTag leftTypeTag = PointableHelper.getTypeTag(leftPointable);
+ ATypeTag rightTypeTag = PointableHelper.getTypeTag(rightPointable);
+
+ if (leftTypeTag != rightTypeTag) {
+ // If types are numeric compare their real values instead
+ if (ATypeHierarchy.isSameTypeDomain(leftTypeTag, rightTypeTag, false)
+ && ATypeHierarchy.getTypeDomain(leftTypeTag) == Domain.NUMERIC) {
+ try {
+ double leftVal = ATypeHierarchy.getDoubleValue(leftPointable.getByteArray(),
+ leftPointable.getStartOffset());
+ double rightVal = ATypeHierarchy.getDoubleValue(rightPointable.getByteArray(),
+ rightPointable.getStartOffset());
+ return (leftVal == rightVal);
+ } catch (HyracksDataException e) {
+ throw new AlgebricksException(e);
+ }
+
+ } else {
+ return false;
+ }
+ }
+
+ Pair<IVisitablePointable, Boolean> arg = new Pair<IVisitablePointable, Boolean>(rightPointable, Boolean.FALSE);
+ // Assess the nested equality
+ leftPointable.accept(equalityVisitor, arg);
+
+ return arg.second;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/DeepEqualityDescriptor.java
----------------------------------------------------------------------
diff --git a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/DeepEqualityDescriptor.java b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/DeepEqualityDescriptor.java
new file mode 100644
index 0000000..e6a4232
--- /dev/null
+++ b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/DeepEqualityDescriptor.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.DataOutput;
+
+import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.pointables.PointableAllocator;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.comparisons.DeepEqualAssessor;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.data.std.api.IDataOutputProvider;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class DeepEqualityDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new DeepEqualityDescriptor();
+ }
+ };
+
+ private static final long serialVersionUID = 1L;
+ private IAType inputTypeLeft;
+ private IAType inputTypeRight;
+
+ public void reset(IAType inTypeLeft, IAType inTypeRight) {
+ this.inputTypeLeft = inTypeLeft;
+ this.inputTypeRight = inTypeRight;
+ }
+
+ @Override
+ public ICopyEvaluatorFactory createEvaluatorFactory(final ICopyEvaluatorFactory[] args) {
+ final ICopyEvaluatorFactory evalFactoryLeft = args[0];
+ final ICopyEvaluatorFactory evalFactoryRight = args[1];
+
+ return new ICopyEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+ private final ISerializerDeserializer boolSerde = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.ABOOLEAN);
+
+ @Override
+ public ICopyEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+ final DataOutput out = output.getDataOutput();
+ final ArrayBackedValueStorage abvsLeft = new ArrayBackedValueStorage();
+ final ICopyEvaluator evalLeft = evalFactoryLeft.createEvaluator(abvsLeft);
+
+ final ArrayBackedValueStorage abvsRight = new ArrayBackedValueStorage();
+ final ICopyEvaluator evalRight = evalFactoryRight.createEvaluator(abvsRight);
+ final DeepEqualAssessor deepEqualAssessor = new DeepEqualAssessor();
+
+ return new ICopyEvaluator() {
+ private final PointableAllocator allocator = new PointableAllocator();
+ private final IVisitablePointable pointableLeft = allocator.allocateFieldValue(inputTypeLeft);
+ private final IVisitablePointable pointableRight = allocator.allocateFieldValue(inputTypeRight);
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+ try {
+ abvsLeft.reset();
+ abvsRight.reset();
+ evalLeft.evaluate(tuple);
+ evalRight.evaluate(tuple);
+ pointableLeft.set(abvsLeft);
+ pointableRight.set(abvsRight);
+
+ // Using deep equality assessment to assess the equality of the two values
+ boolean isEqual = deepEqualAssessor.isEqual(pointableLeft, pointableRight);
+ ABoolean result = isEqual ? ABoolean.TRUE : ABoolean.FALSE;
+
+ boolSerde.serialize(result, out);
+ } catch (Exception ioe) {
+ throw new AlgebricksException(ioe);
+ }
+ }
+ };
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return AsterixBuiltinFunctions.DEEP_EQUAL;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java
----------------------------------------------------------------------
diff --git a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java
new file mode 100644
index 0000000..24e66f2
--- /dev/null
+++ b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import org.apache.hyracks.data.std.api.IMutableValueStorage;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
+import org.apache.hyracks.util.string.UTF8StringWriter;
+
+/**
+ * An utility class for some frequently used methods like checking the equality between two pointables (binary values)
+ * (e.g., field names), string value of a fieldname pointable, getting the typetag of a pointable, etc.
+ * Note: To get the typetag of a fieldvalue (i) in a record, it is recommended to use the getFieldTypeTags().get(i)
+ * method rather than getting it from fhe field value itself.
+ */
+
+public class PointableHelper {
+ private static final IBinaryComparator STRING_BINARY_COMPARATOR = PointableBinaryComparatorFactory.of(
+ UTF8StringPointable.FACTORY).createBinaryComparator();
+ private final UTF8StringWriter utf8Writer;
+
+ public PointableHelper() {
+ utf8Writer = new UTF8StringWriter();
+ }
+
+ public static int compareStringBinValues(IValueReference a, IValueReference b) throws HyracksDataException {
+ // start+1 and len-1 due to type tag ignore (only interested in String value)
+ return STRING_BINARY_COMPARATOR.compare(a.getByteArray(), a.getStartOffset() + 1, a.getLength() - 1,
+ b.getByteArray(), b.getStartOffset() + 1, b.getLength() - 1);
+ }
+
+ public static boolean isEqual(IValueReference a, IValueReference b) throws HyracksDataException {
+ return (compareStringBinValues(a, b) == 0);
+ }
+
+ public static boolean byteArrayEqual(IValueReference valueRef1, IValueReference valueRef2) {
+ return byteArrayEqual(valueRef1, valueRef2, 3);
+ }
+
+ public static boolean byteArrayEqual(IValueReference valueRef1, IValueReference valueRef2, int dataOffset) {
+ if (valueRef1 == null || valueRef2 == null) {
+ return false;
+ }
+ if (valueRef1 == valueRef2) {
+ return true;
+ }
+
+ int length1 = valueRef1.getLength();
+ int length2 = valueRef2.getLength();
+
+ if (length1 != length2) {
+ return false;
+ }
+
+ byte[] bytes1 = valueRef1.getByteArray();
+ byte[] bytes2 = valueRef2.getByteArray();
+ int start1 = valueRef1.getStartOffset() + dataOffset;
+ int start2 = valueRef2.getStartOffset() + dataOffset;
+
+ int end = start1 + length1 - dataOffset;
+
+ for (int i = start1, j = start2; i < end; i++, j++) {
+ if (bytes1[i] != bytes2[j]) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ public static boolean sameType(ATypeTag typeTag, IVisitablePointable visitablePointable) {
+ return (getTypeTag(visitablePointable) == typeTag);
+ }
+
+ public static ATypeTag getTypeTag(IValueReference visitablePointable) {
+ byte[] bytes = visitablePointable.getByteArray();
+ int s = visitablePointable.getStartOffset();
+ return EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes[s]);
+ }
+
+ /**
+ * @param str
+ * The input string
+ * @param vs
+ * The storage buffer
+ * @param writeTag
+ * Specifying whether a tag for the string should also be written
+ * @throws AlgebricksException
+ */
+ public void serializeString(String str, IMutableValueStorage vs, boolean writeTag) throws AsterixException {
+ vs.reset();
+ try {
+ DataOutput output = vs.getDataOutput();
+ if (writeTag) {
+ output.write(ATypeTag.STRING.serialize());
+ }
+ utf8Writer.writeUTF8(str, output);
+ } catch (IOException e) {
+ throw new AsterixException("Could not serialize " + str);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddFieldsDescriptor.java
----------------------------------------------------------------------
diff --git a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddFieldsDescriptor.java b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddFieldsDescriptor.java
new file mode 100644
index 0000000..c26daca
--- /dev/null
+++ b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddFieldsDescriptor.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.evaluators.functions.records;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.asterix.builders.RecordBuilder;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.dataflow.data.nontagged.comparators.ListItemBinaryComparatorFactory;
+import org.apache.asterix.dataflow.data.nontagged.hash.ListItemBinaryHashFunctionFactory;
+import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import org.apache.asterix.om.base.ANull;
+import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.pointables.AListVisitablePointable;
+import org.apache.asterix.om.pointables.ARecordVisitablePointable;
+import org.apache.asterix.om.pointables.PointableAllocator;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.typecomputer.impl.TypeComputerUtils;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.runtime.RuntimeRecordTypeInfo;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.BinaryHashMap;
+import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
+import org.apache.hyracks.api.dataflow.value.IBinaryHashFunction;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IDataOutputProvider;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class RecordAddFieldsDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new RecordAddFieldsDescriptor();
+ }
+ };
+ private static final long serialVersionUID = 1L;
+ private ARecordType outRecType;
+ private ARecordType inRecType;
+ private AOrderedListType inListType;
+ private IAType inputFieldListItemType;
+
+ public void reset(IAType outType, IAType inType0, IAType inType1) {
+ outRecType = TypeComputerUtils.extractRecordType(outType);
+ inRecType = TypeComputerUtils.extractRecordType(inType0);
+ inListType = TypeComputerUtils.extractOrderedListType(inType1);
+ inputFieldListItemType = inListType.getItemType();
+ if (inputFieldListItemType == null || inputFieldListItemType.getTypeTag() == ATypeTag.ANY) {
+ inputFieldListItemType = DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+ }
+ }
+
+ @Override
+ public ICopyEvaluatorFactory createEvaluatorFactory(final ICopyEvaluatorFactory[] args) throws AlgebricksException {
+ return new ICopyEvaluatorFactory() {
+
+ private static final long serialVersionUID = 1L;
+ private final byte SER_NULL_TYPE_TAG = ATypeTag.NULL.serialize();
+ private final byte SER_ORDERED_TYPE_TAG = ATypeTag.ORDEREDLIST.serialize();
+ private final byte SER_RECORD_TYPE_TAG = ATypeTag.RECORD.serialize();
+
+ @SuppressWarnings("unchecked")
+ private final ISerializerDeserializer<ANull> nullSerDe = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.ANULL);
+
+ @Override
+ public ICopyEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+ final PointableAllocator allocator = new PointableAllocator();
+ final IVisitablePointable vp0 = allocator.allocateRecordValue(inRecType);
+ final IVisitablePointable vp1 = allocator.allocateListValue(inListType);
+
+ final ArrayBackedValueStorage abvs0 = new ArrayBackedValueStorage();
+ final ArrayBackedValueStorage abvs1 = new ArrayBackedValueStorage();
+
+ final ICopyEvaluator eval0 = args[0].createEvaluator(abvs0);
+ final ICopyEvaluator eval1 = args[1].createEvaluator(abvs1);
+
+ final ArrayBackedValueStorage fieldNamePointable = new ArrayBackedValueStorage();
+ final ArrayBackedValueStorage fieldValuePointer = new ArrayBackedValueStorage();
+ final PointableHelper pointableHelper = new PointableHelper();
+ try {
+ pointableHelper.serializeString("field-name", fieldNamePointable, true);
+ pointableHelper.serializeString("field-value", fieldValuePointer, true);
+ } catch (AsterixException e) {
+ throw new AlgebricksException(e);
+ }
+
+ return new ICopyEvaluator() {
+ public static final int TABLE_FRAME_SIZE = 32768; // the default 32k frame size
+ public static final int TABLE_SIZE = 100; // the default 32k frame size
+ private final RecordBuilder recordBuilder = new RecordBuilder();
+ private final RuntimeRecordTypeInfo requiredRecordTypeInfo = new RuntimeRecordTypeInfo();
+
+ private final IBinaryHashFunction putHashFunc = ListItemBinaryHashFunctionFactory.INSTANCE
+ .createBinaryHashFunction();
+ private final IBinaryHashFunction getHashFunc = ListItemBinaryHashFunctionFactory.INSTANCE
+ .createBinaryHashFunction();
+ private final BinaryHashMap.BinaryEntry keyEntry = new BinaryHashMap.BinaryEntry();
+ private final BinaryHashMap.BinaryEntry valEntry = new BinaryHashMap.BinaryEntry();
+ private final IVisitablePointable tempValReference = allocator.allocateEmpty();
+ private final IBinaryComparator cmp = ListItemBinaryComparatorFactory.INSTANCE
+ .createBinaryComparator();
+ private BinaryHashMap hashMap = new BinaryHashMap(TABLE_SIZE, TABLE_FRAME_SIZE, putHashFunc,
+ getHashFunc, cmp);
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+ recordBuilder.reset(outRecType);
+ requiredRecordTypeInfo.reset(outRecType);
+ abvs0.reset();
+ abvs1.reset();
+
+ eval0.evaluate(tuple);
+ eval1.evaluate(tuple);
+
+ if (abvs0.getByteArray()[0] == SER_NULL_TYPE_TAG
+ || abvs1.getByteArray()[0] == SER_NULL_TYPE_TAG) {
+ try {
+ nullSerDe.serialize(ANull.NULL, output.getDataOutput());
+ } catch (HyracksDataException e) {
+ throw new AlgebricksException(e);
+ }
+ return;
+ }
+
+ // Make sure we get a valid record
+ if (abvs0.getByteArray()[0] != SER_RECORD_TYPE_TAG) {
+ throw new AlgebricksException("Expected an ordederlist of type " + inRecType + " but "
+ + "got "
+ + EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(abvs0.getByteArray()[0]));
+ }
+
+ // Make sure we get a valid list
+ if (abvs1.getByteArray()[0] != SER_ORDERED_TYPE_TAG) {
+ throw new AlgebricksException("Expected an ordederlist of type " + inListType + " but "
+ + "got "
+ + EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(abvs1.getByteArray()[0]));
+ }
+
+ vp0.set(abvs0);
+ vp1.set(abvs1);
+
+ try {
+ ARecordVisitablePointable recordPointable = (ARecordVisitablePointable) vp0;
+ AListVisitablePointable listPointable = (AListVisitablePointable) vp1;
+
+ // Initialize our hashmap
+ int tableSize = recordPointable.getFieldNames().size() + listPointable.getItems().size();
+ // Construct a new hash table only if table size is larger than the default
+ // Thus avoiding unnecessary object construction
+ if (hashMap == null || tableSize > TABLE_SIZE) {
+ hashMap = new BinaryHashMap(tableSize, TABLE_FRAME_SIZE, putHashFunc, getHashFunc, cmp);
+ } else {
+ hashMap.clear();
+ }
+ addFields(recordPointable, listPointable);
+ recordBuilder.write(output.getDataOutput(), true);
+ } catch (IOException | AsterixException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ private void addFields(ARecordVisitablePointable inputRecordPointer,
+ AListVisitablePointable listPointable) throws AlgebricksException {
+ List<IVisitablePointable> inputRecordFieldNames = inputRecordPointer.getFieldNames();
+ List<IVisitablePointable> inputRecordFieldValues = inputRecordPointer.getFieldValues();
+ List<IVisitablePointable> inputFields = listPointable.getItems();
+ IVisitablePointable namePointable = null;
+ IVisitablePointable valuePointable = null;
+ int numInputRecordFields = inputRecordFieldNames.size();
+
+ try {
+ // Add original record without duplicate checking
+ for (int i = 0; i < numInputRecordFields; ++i) {
+ IVisitablePointable fnp = inputRecordFieldNames.get(i);
+ IVisitablePointable fvp = inputRecordFieldValues.get(i);
+ int pos = requiredRecordTypeInfo.getFieldIndex(fnp.getByteArray(),
+ fnp.getStartOffset() + 1, fnp.getLength() - 1);
+ if (pos >= 0) {
+ recordBuilder.addField(pos, fvp);
+ } else {
+ recordBuilder.addField(fnp, fvp);
+ }
+ keyEntry.set(fnp.getByteArray(), fnp.getStartOffset(), fnp.getLength());
+ valEntry.set(fvp.getByteArray(), fvp.getStartOffset(), fvp.getLength());
+ hashMap.put(keyEntry, valEntry);
+ }
+
+ // Get the fields from a list of records
+ for (int i = 0; i < inputFields.size(); i++) {
+ if (!PointableHelper.sameType(ATypeTag.RECORD, inputFields.get(i))) {
+ throw new AsterixException("Expected list of record, got "
+ + PointableHelper.getTypeTag(inputFields.get(i)));
+ }
+ List<IVisitablePointable> names = ((ARecordVisitablePointable) inputFields.get(i))
+ .getFieldNames();
+ List<IVisitablePointable> values = ((ARecordVisitablePointable) inputFields.get(i))
+ .getFieldValues();
+
+ // Get name and value of the field to be added
+ // Use loop to account for the cases where users switches the order of the fields
+ IVisitablePointable fieldName;
+ for (int j = 0; j < names.size(); j++) {
+ fieldName = names.get(j);
+ // if fieldName is "field-name" then read the name
+ if (PointableHelper.byteArrayEqual(fieldNamePointable, fieldName)) {
+ namePointable = values.get(j);
+ } else { // otherwise the fieldName is "field-value". Thus, read the value
+ valuePointable = values.get(j);
+ }
+ }
+
+ if (namePointable == null || valuePointable == null) {
+ throw new AlgebricksException("Trying to add a null field name or field value");
+ }
+
+ // Check that the field being added is a valid field
+ int pos = requiredRecordTypeInfo.getFieldIndex(namePointable.getByteArray(),
+ namePointable.getStartOffset() + 1, namePointable.getLength() - 1);
+
+ keyEntry.set(namePointable.getByteArray(), namePointable.getStartOffset(),
+ namePointable.getLength());
+ // Check if already in our built record
+ BinaryHashMap.BinaryEntry entry = hashMap.get(keyEntry);
+ if (entry != null) {
+ tempValReference.set(entry.buf, entry.off, entry.len);
+ // If value is not equal throw conflicting duplicate field, otherwise ignore
+ if (!PointableHelper.byteArrayEqual(valuePointable, tempValReference)) {
+ throw new AlgebricksException("Conflicting duplicate field found.");
+ }
+ } else {
+ if (pos > -1) {
+ recordBuilder.addField(pos, valuePointable);
+ } else {
+ recordBuilder.addField(namePointable, valuePointable);
+ }
+ valEntry.set(valuePointable.getByteArray(), valuePointable.getStartOffset(),
+ valuePointable.getLength());
+ hashMap.put(keyEntry, valEntry);
+ }
+ }
+ } catch (AsterixException | HyracksDataException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+ };
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return AsterixBuiltinFunctions.ADD_FIELDS;
+ }
+}