You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by dl...@apache.org on 2021/06/30 22:05:23 UTC
[asterixdb] branch master updated: [NO ISSUE][IDX] Refactoring of
array index code.
This is an automated email from the ASF dual-hosted git repository.
dlych pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new e381b2c [NO ISSUE][IDX] Refactoring of array index code.
e381b2c is described below
commit e381b2cbdadc3e33a488678e3b2384a00d97f301
Author: ggalvizo <gg...@uci.edu>
AuthorDate: Tue Jun 29 15:27:30 2021 -0700
[NO ISSUE][IDX] Refactoring of array index code.
- user model changes: no
- storage format changes: no
- interface changes: no
Refactor: no longer using depth indicators, rather using UNNEST flags with
flattened field names. We can no longer have back-to-back UNNESTs (i.e. arrays
within arrays without an ASSIGN intermediate), as no such array index
can be created..
Change-Id: Id337d1032796e1d1c2ce68ea2861b4a81dd19aa5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12063
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Glenn Galvizo <gg...@uci.edu>
---
.../IntroduceSecondaryIndexInsertDeleteRule.java | 26 +--
.../optimizer/rules/am/AccessMethodUtils.java | 1 -
.../optimizer/rules/am/ArrayBTreeAccessMethod.java | 1 -
.../ArrayBTreeResourceFactoryProvider.java | 9 +-
.../asterix/metadata/utils/ArrayIndexUtil.java | 214 ++++++++++++---------
.../asterix/metadata/utils/KeyFieldTypeUtil.java | 4 +-
.../SecondaryArrayIndexBTreeOperationsHelper.java | 62 +++---
.../apache/asterix/metadata/utils/TypeUtil.java | 71 +++----
8 files changed, 197 insertions(+), 191 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index c3859be..81c6c2a 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -728,10 +728,10 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
// Walk the array path.
List<String> flatFirstFieldName = ArrayIndexUtil.getFlattenedKeyFieldNames(
workingElement.getUnnestList(), workingElement.getProjectList().get(0));
- List<Integer> firstArrayIndicator = ArrayIndexUtil
- .getArrayDepthIndicator(workingElement.getUnnestList(), workingElement.getProjectList().get(0));
+ List<Boolean> firstUnnestFlags = ArrayIndexUtil.getUnnestFlags(workingElement.getUnnestList(),
+ workingElement.getProjectList().get(0));
ArrayIndexUtil.walkArrayPath((isOpenOrNestedField) ? null : recordType, flatFirstFieldName,
- firstArrayIndicator, branchCreator);
+ firstUnnestFlags, branchCreator);
// For all other elements in the PROJECT list, add an assign.
for (int j = 1; j < workingElement.getProjectList().size(); j++) {
@@ -1054,25 +1054,19 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
@Override
public void executeActionOnEachArrayStep(ARecordType startingStepRecordType, IAType workingType,
- List<String> fieldName, boolean isFirstArrayStep, boolean isFirstUnnestInStep,
- boolean isLastUnnestInIntermediateStep) throws AlgebricksException {
+ List<String> fieldName, boolean isFirstArrayStep, boolean isLastUnnestInIntermediateStep)
+ throws AlgebricksException {
if (!isFirstWalk) {
// We have already built the UNNEST path, do not build again.
return;
}
+ // Get the field we want to UNNEST from our record.
ILogicalExpression accessToUnnestVar;
- if (isFirstUnnestInStep) {
- // This is the first UNNEST step. Get the field we want to UNNEST from our record.
- accessToUnnestVar = (startingStepRecordType != null)
- ? getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()),
- startingStepRecordType.getFieldIndex(fieldName.get(0)), fieldName)
- : getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()), -1, fieldName);
- } else {
- // This is the second+ UNNEST step. Refer back to the previously unnested variable.
- accessToUnnestVar = new VariableReferenceExpression(this.lastFieldVars.get(0));
- this.lastFieldVars.clear();
- }
+ accessToUnnestVar = (startingStepRecordType != null)
+ ? getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()),
+ startingStepRecordType.getFieldIndex(fieldName.get(0)), fieldName)
+ : getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()), -1, fieldName);
UnnestingFunctionCallExpression scanCollection = new UnnestingFunctionCallExpression(
BuiltinFunctions.getBuiltinFunctionInfo(BuiltinFunctions.SCAN_COLLECTION),
Collections.singletonList(new MutableObject<>(accessToUnnestVar)));
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index c78e89f..3f4d3f2 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -3060,7 +3060,6 @@ public class AccessMethodUtils {
MutableInt fieldSource = new MutableInt(0);
ARecordType workingRecordType = subTree.getRecordType();
- // TODO: (GLENN) Refactor this to use ArrayIndexUtil.
// Iterate through our array index structure. We must match the depth and field names for the caller's variable
// to qualify for an array-index optimization.
LogicalVariable varFromParent = assignVar;
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/ArrayBTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/ArrayBTreeAccessMethod.java
index 653d15b..d364b53 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/ArrayBTreeAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/ArrayBTreeAccessMethod.java
@@ -179,7 +179,6 @@ public class ArrayBTreeAccessMethod extends BTreeAccessMethod {
@Override
protected IAType getIndexedKeyType(Index.IIndexDetails chosenIndexDetails, int keyPos) throws CompilationException {
- // TODO (GLENN): This assumes a flattened key list. Refactor / clarify this when removing depth indicators.
Index.ArrayIndexDetails arrayIndexDetails = (Index.ArrayIndexDetails) chosenIndexDetails;
int elementPos = 0;
for (Index.ArrayIndexElement e : arrayIndexDetails.getElementList()) {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ArrayBTreeResourceFactoryProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ArrayBTreeResourceFactoryProvider.java
index 5654d16..c0bdc75 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ArrayBTreeResourceFactoryProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ArrayBTreeResourceFactoryProvider.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.metadata.declared;
-import java.util.List;
import java.util.Map;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
@@ -134,10 +133,8 @@ public class ArrayBTreeResourceFactoryProvider implements IResourceFactoryProvid
sourceType = metaType;
}
for (int i = 0; i < e.getProjectList().size(); i++) {
- List<String> project = e.getProjectList().get(i);
Pair<IAType, Boolean> keyTypePair = ArrayIndexUtil.getNonNullableOpenFieldType(e.getTypeList().get(i),
- ArrayIndexUtil.getFlattenedKeyFieldNames(e.getUnnestList(), project), sourceType,
- ArrayIndexUtil.getArrayDepthIndicator(e.getUnnestList(), project));
+ e.getUnnestList(), e.getProjectList().get(i), sourceType);
IAType keyType = keyTypePair.first;
secondaryTypeTraits[secondaryTypeTraitPos++] = typeTraitProvider.getTypeTrait(keyType);
}
@@ -175,10 +172,8 @@ public class ArrayBTreeResourceFactoryProvider implements IResourceFactoryProvid
sourceType = metaType;
}
for (int i = 0; i < e.getProjectList().size(); i++) {
- List<String> project = e.getProjectList().get(i);
Pair<IAType, Boolean> keyTypePair = ArrayIndexUtil.getNonNullableOpenFieldType(e.getTypeList().get(i),
- ArrayIndexUtil.getFlattenedKeyFieldNames(e.getUnnestList(), project), sourceType,
- ArrayIndexUtil.getArrayDepthIndicator(e.getUnnestList(), project));
+ e.getUnnestList(), e.getProjectList().get(i), sourceType);
IAType keyType = keyTypePair.first;
secondaryCmpFactories[secondaryCmpFactoriesPos++] =
cmpFactoryProvider.getBinaryComparatorFactory(keyType, true);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
index f4dfe56..0778ad9 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
@@ -39,14 +39,18 @@ import org.apache.hyracks.algebricks.common.utils.Pair;
public class ArrayIndexUtil {
/**
- * @deprecated Use the project + unnest scheme instead of array indicators.
+ * Similar function to Index's "getSubFieldType", but accounts for array fields as well.
*/
- public static IAType getSubFieldInArrayType(ARecordType recordType, List<String> subFieldName,
- List<Integer> arrayDepthIndicators) throws AlgebricksException {
- IAType subType = recordType.getFieldType(subFieldName.get(0));
- for (int i = 1; i < subFieldName.size(); i++) {
+ public static IAType getSubFieldType(ARecordType recordType, List<List<String>> unnestList,
+ List<String> projectList) throws AlgebricksException {
+ List<String> flattenedFieldName = ArrayIndexUtil.getFlattenedKeyFieldNames(unnestList, projectList);
+ List<Boolean> unnestFlags = ArrayIndexUtil.getUnnestFlags(unnestList, projectList);
+ IAType subType = recordType.getFieldType(flattenedFieldName.get(0));
+
+ for (int i = 1; i < flattenedFieldName.size(); i++) {
if (subType == null) {
return null;
+
} else if (subType.getTypeTag().equals(ATypeTag.UNION)) {
// Support enforced types here.
subType = ((AUnionType) subType).getActualType();
@@ -56,31 +60,30 @@ public class ArrayIndexUtil {
"Field accessor is not defined for values of type " + subType.getTypeTag());
}
}
- if (subType.getTypeTag().equals(ATypeTag.OBJECT) && arrayDepthIndicators.get(i - 1) == 0) {
- subType = ((ARecordType) subType).getFieldType(subFieldName.get(i));
+
+ if (subType.getTypeTag().equals(ATypeTag.OBJECT) && !unnestFlags.get(i - 1)) {
+ subType = ((ARecordType) subType).getFieldType(flattenedFieldName.get(i));
+
} else if ((subType.getTypeTag().equals(ATypeTag.ARRAY) || subType.getTypeTag().equals(ATypeTag.MULTISET))
- && arrayDepthIndicators.get(i - 1) > 0) {
- for (int j = 0; j < arrayDepthIndicators.get(i - 1); j++) {
- subType = TypeComputeUtils.extractListItemType(subType);
- }
- subType = (subType != null) ? ((ARecordType) subType).getFieldType(subFieldName.get(i)) : null;
+ && unnestFlags.get(i - 1)) {
+ subType = TypeComputeUtils.extractListItemType(subType);
+ subType = (subType != null) ? ((ARecordType) subType).getFieldType(flattenedFieldName.get(i)) : null;
+
} else {
throw new AsterixException(ErrorCode.COMPILATION_ERROR,
- (arrayDepthIndicators.get(i - 1) > 0)
- ? "Object type given, but array depth indicator is " + "non-zero."
- : "Array/multiset type given, but array depth indicator is zero.");
+ unnestFlags.get(i - 1) ? "Object type given, but unnest flag is also raised."
+ : "Array/multiset type given, but unnest flag is lowered.");
}
}
- if (subType != null && arrayDepthIndicators.get(arrayDepthIndicators.size() - 1) > 0) {
+
+ if (subType != null && unnestFlags.get(unnestFlags.size() - 1)) {
// If the end field is an array, we must extract the list item here as well.
- for (int j = 0; j < arrayDepthIndicators.get(arrayDepthIndicators.size() - 1); j++) {
- if (subType instanceof AbstractCollectionType) {
- subType = TypeComputeUtils.extractListItemType(subType);
- } else {
- throw new AsterixException(ErrorCode.COMPILATION_ERROR,
- "Array type expected for last term, but given: "
- + ((subType != null) ? subType.getTypeTag() : "null"));
- }
+ if (subType instanceof AbstractCollectionType) {
+ subType = TypeComputeUtils.extractListItemType(subType);
+
+ } else {
+ throw new AsterixException(ErrorCode.COMPILATION_ERROR,
+ "Array type expected for last term, but given: " + subType.getTypeTag());
}
}
return subType;
@@ -88,15 +91,18 @@ public class ArrayIndexUtil {
/**
* Given a path of complex types (i.e. lists + records), determine the nullability of the field.
- * @deprecated Use the project + unnest scheme instead of array indicators.
*/
- public static boolean isSubFieldNullable(ARecordType recordType, List<String> subFieldName,
- List<Integer> arrayIndicators) throws AlgebricksException {
- IAType subType = recordType.getFieldType(subFieldName.get(0));
- for (int i = 1; i < subFieldName.size(); i++) {
+ public static boolean isSubFieldNullable(ARecordType recordType, List<List<String>> unnestList,
+ List<String> projectList) throws AlgebricksException {
+ List<String> flattenedFieldName = ArrayIndexUtil.getFlattenedKeyFieldNames(unnestList, projectList);
+ List<Boolean> unnestFlags = ArrayIndexUtil.getUnnestFlags(unnestList, projectList);
+ IAType subType = recordType.getFieldType(flattenedFieldName.get(0));
+
+ for (int i = 1; i < flattenedFieldName.size(); i++) {
if (subType == null) {
return true;
}
+
if (subType.getTypeTag().equals(ATypeTag.UNION)) {
if (NonTaggedFormatUtil.isOptional(subType)) {
return true;
@@ -109,12 +115,12 @@ public class ArrayIndexUtil {
}
if (subType instanceof ARecordType) {
- subType = ((ARecordType) subType).getFieldType(subFieldName.get(i));
- } else if (subType instanceof AbstractCollectionType && arrayIndicators.get(i - 1) > 0) {
- for (int j = 0; j < arrayIndicators.get(i - 1); j++) {
- subType = TypeComputeUtils.extractListItemType(subType);
- }
- subType = (subType != null) ? ((ARecordType) subType).getFieldType(subFieldName.get(i)) : null;
+ subType = ((ARecordType) subType).getFieldType(flattenedFieldName.get(i));
+
+ } else if (subType instanceof AbstractCollectionType && unnestFlags.get(i - 1)) {
+ subType = TypeComputeUtils.extractListItemType(subType);
+ subType = (subType != null) ? ((ARecordType) subType).getFieldType(flattenedFieldName.get(i)) : null;
+
} else {
throw CompilationException.create(ErrorCode.COMPILATION_ILLEGAL_STATE,
"Illegal field type " + subType.getTypeTag() + " when checking field nullability");
@@ -125,32 +131,37 @@ public class ArrayIndexUtil {
/**
* Similar function to Index's "getNonNullableOpenFieldType", but accounts for array fields as well.
- * @deprecated Use the project + unnest scheme instead of array indicators.
*/
- public static Pair<IAType, Boolean> getNonNullableOpenFieldType(IAType fieldType, List<String> fieldName,
- ARecordType recType, List<Integer> arrayIndicators) throws AlgebricksException {
+ public static Pair<IAType, Boolean> getNonNullableOpenFieldType(IAType fieldType, List<List<String>> unnestList,
+ List<String> projectList, ARecordType recType) throws AlgebricksException {
Pair<IAType, Boolean> keyPairType = null;
IAType subType = recType;
boolean nullable = false;
- for (int i = 0; i < fieldName.size(); i++) {
+
+ List<String> flattenedFieldName = ArrayIndexUtil.getFlattenedKeyFieldNames(unnestList, projectList);
+ List<Boolean> unnestFlags = ArrayIndexUtil.getUnnestFlags(unnestList, projectList);
+ for (int i = 0; i < flattenedFieldName.size(); i++) {
if (subType instanceof AUnionType) {
nullable = nullable || ((AUnionType) subType).isUnknownableType();
subType = ((AUnionType) subType).getActualType();
}
if (subType instanceof ARecordType) {
- subType = ((ARecordType) subType).getFieldType(fieldName.get(i));
+ subType = ((ARecordType) subType).getFieldType(flattenedFieldName.get(i));
+
} else if ((subType instanceof AOrderedListType || subType instanceof AUnorderedListType)
- && arrayIndicators.get(i - 1) > 0) {
- for (int j = 0; j < arrayIndicators.get(i - 1); j++) {
- subType = TypeComputeUtils.extractListItemType(subType);
- }
+ && unnestFlags.get(i - 1)) {
+ subType = TypeComputeUtils.extractListItemType(subType);
if (subType instanceof ARecordType) {
- subType = ((ARecordType) subType).getFieldType(fieldName.get(i));
+ subType = ((ARecordType) subType).getFieldType(flattenedFieldName.get(i));
+
} else {
- throw AsterixException.create(ErrorCode.COMPILATION_ILLEGAL_STATE, "Unexpected type " + fieldType);
+ throw AsterixException.create(ErrorCode.COMPILATION_ILLEGAL_STATE,
+ "Unexpected type " + subType + ", expected record.");
}
+
} else {
- throw AsterixException.create(ErrorCode.COMPILATION_ILLEGAL_STATE, "Unexpected type " + fieldType);
+ throw AsterixException.create(ErrorCode.COMPILATION_ILLEGAL_STATE,
+ "Unexpected type " + subType + ", expected record, array, or multi-set.");
}
if (subType == null) {
@@ -158,18 +169,20 @@ public class ArrayIndexUtil {
break;
}
}
+
if (subType != null) {
- IAType keyType = ArrayIndexUtil.getSubFieldInArrayType(recType, fieldName, arrayIndicators);
+ IAType keyType = ArrayIndexUtil.getSubFieldType(recType, unnestList, projectList);
Pair<IAType, Boolean> pair = Index.getNonNullableType(keyType);
- pair.second = pair.second || ArrayIndexUtil.isSubFieldNullable(recType, fieldName, arrayIndicators);
+ pair.second = pair.second || ArrayIndexUtil.isSubFieldNullable(recType, unnestList, projectList);
keyPairType = pair;
}
+
keyPairType.second = keyPairType.second || nullable;
return keyPairType;
}
/**
- * @deprecated Use new unnestList and projectList scheme.
+ * @return The concatenation of the unnest list fields and the project field (for use in creating a unique name).
*/
public static List<String> getFlattenedKeyFieldNames(List<List<String>> unnestList, List<String> projectList) {
if (unnestList == null) {
@@ -188,6 +201,41 @@ public class ArrayIndexUtil {
}
/**
+ * @return Mapping to the flattened key field names, determine where the UNNESTs occur.
+ */
+ public static List<Boolean> getUnnestFlags(List<List<String>> unnestList, List<String> projectList) {
+ if (unnestList.isEmpty()) {
+ // A simple element has no UNNEST flags raised..
+ List<Boolean> unnestFlags = new ArrayList<>();
+ for (String ignored : projectList) {
+ unnestFlags.add(false);
+ }
+ return unnestFlags;
+
+ } else {
+ List<Boolean> unnestFlagsPrefix = new ArrayList<>();
+ for (List<String> unnestField : unnestList) {
+ for (int i = 0; i < unnestField.size() - 1; i++) {
+ unnestFlagsPrefix.add(false);
+ }
+ unnestFlagsPrefix.add(true);
+ }
+
+ if (projectList == null) {
+ // Stop here. The prefix is the flag vector itself.
+ return unnestFlagsPrefix;
+
+ } else {
+ List<Boolean> unnestFlags = new ArrayList<>(unnestFlagsPrefix);
+ for (int i = 0; i < projectList.size(); i++) {
+ unnestFlags.add(false);
+ }
+ return unnestFlags;
+ }
+ }
+ }
+
+ /**
* @deprecated Use new unnestList and projectList scheme.
*/
public static List<Integer> getArrayDepthIndicator(List<List<String>> unnestList, List<String> projectList) {
@@ -244,18 +292,14 @@ public class ArrayIndexUtil {
}
/**
- * Given the {@code Index}'s representation of an array path (i.e. a concatenation of record paths, with array
- * steps specified in depths corresponding to an index in the aforementioned record path array), traverse each
- * distinct record path and invoke the appropriate commands for each scenario.
- * <p>
- * Here, we keep track of the record/list type at each step and give this to each command.
+ * Traverse each distinct record path and invoke the appropriate commands for each scenario. Here, we keep track
+ * of the record/list type at each step and give this to each command.
*/
public static void walkArrayPath(ARecordType baseRecordType, List<String> flattenedFieldName,
- List<Integer> flattenedDepthIndicators, TypeTrackerCommandExecutor commandExecutor)
- throws AlgebricksException {
- ArrayPath arrayPath = new ArrayPath(flattenedFieldName, flattenedDepthIndicators).invoke();
+ List<Boolean> unnestFlags, TypeTrackerCommandExecutor commandExecutor) throws AlgebricksException {
+ ArrayPath arrayPath = new ArrayPath(flattenedFieldName, unnestFlags).invoke();
List<List<String>> fieldNamesPerArray = arrayPath.fieldNamesPerArray;
- List<Integer> depthOfArraySteps = arrayPath.depthOfArraySteps;
+ List<Boolean> unnestFlagsPerArray = arrayPath.unnestFlagsPerArray;
// If we are given no base record type, then we do not need to keep track of the record type. We are solely
// using this walk for its flags.
@@ -275,7 +319,7 @@ public class ArrayIndexUtil {
startingStepRecordType).first;
}
- for (int j = 0; j < depthOfArraySteps.get(i); j++) {
+ if (unnestFlagsPerArray.get(i)) {
if (isTrackingType) {
workingType = TypeComputeUtils.extractListItemType(workingType);
if (workingType == null) {
@@ -284,17 +328,14 @@ public class ArrayIndexUtil {
}
}
boolean isFirstArrayStep = i == 0;
- boolean isFirstUnnestInStep = j == 0;
- boolean isLastUnnestInIntermediateStep =
- j == depthOfArraySteps.get(i) - 1 && i < fieldNamesPerArray.size() - 1;
+ boolean isLastUnnestInIntermediateStep = i < fieldNamesPerArray.size() - 1;
commandExecutor.executeActionOnEachArrayStep(startingStepRecordType, workingType,
- fieldNamesPerArray.get(i), isFirstArrayStep, isFirstUnnestInStep,
- isLastUnnestInIntermediateStep);
+ fieldNamesPerArray.get(i), isFirstArrayStep, isLastUnnestInIntermediateStep);
}
if (i == fieldNamesPerArray.size() - 1) {
- boolean requiresOnlyOneUnnest = depthOfArraySteps.stream().reduce(0, Integer::sum).equals(1);
- boolean isNonArrayStep = depthOfArraySteps.get(i) == 0;
+ boolean requiresOnlyOneUnnest = fieldNamesPerArray.size() == 1;
+ boolean isNonArrayStep = !unnestFlagsPerArray.get(i);
commandExecutor.executeActionOnFinalArrayStep(startingStepRecordType, fieldNamesPerArray.get(i),
isNonArrayStep, requiresOnlyOneUnnest);
}
@@ -302,28 +343,25 @@ public class ArrayIndexUtil {
}
/**
- * Given the {@code Index}'s representation of an array path (i.e. a concatenation of record paths, with array
- * steps specified in depths corresponding to an index in the aforementioned record path array), traverse each
- * distinct record path and invoke the appropriate commands for each scenario.
- * <p>
- * Here, we keep track of the total number of actions performed and give this to each command.
+ * Traverse each distinct record path and invoke the appropriate commands for each scenario. Here, we keep track
+ * of the total number of actions performed and give this to each command.
*/
- public static void walkArrayPath(List<String> flattenedFieldName, List<Integer> flattenedDepthIndicators,
+ public static void walkArrayPath(List<String> flattenedFieldName, List<Boolean> unnestFlags,
ActionCounterCommandExecutor commandExecutor) throws AlgebricksException {
- ArrayPath arrayPath = new ArrayPath(flattenedFieldName, flattenedDepthIndicators).invoke();
+ ArrayPath arrayPath = new ArrayPath(flattenedFieldName, unnestFlags).invoke();
List<List<String>> fieldNamesPerArray = arrayPath.fieldNamesPerArray;
- List<Integer> depthOfArraySteps = arrayPath.depthOfArraySteps;
+ List<Boolean> unnestFlagsPerArray = arrayPath.unnestFlagsPerArray;
int numberOfActionsPerformed = 0;
for (int i = 0; i < fieldNamesPerArray.size(); i++) {
- int unnestLevel = depthOfArraySteps.get(i);
+ boolean isUnnestFlagRaised = unnestFlagsPerArray.get(i);
if (i == 0) {
commandExecutor.executeActionOnFirstArrayStep();
numberOfActionsPerformed++;
- unnestLevel--;
+ isUnnestFlagRaised = false;
}
- for (int j = 0; j < unnestLevel; j++) {
+ if (isUnnestFlagRaised) {
commandExecutor.executeActionOnIntermediateArrayStep(numberOfActionsPerformed++);
}
@@ -343,8 +381,8 @@ public class ArrayIndexUtil {
public interface TypeTrackerCommandExecutor {
void executeActionOnEachArrayStep(ARecordType startingStepRecordType, IAType workingType,
- List<String> fieldName, boolean isFirstArrayStep, boolean isFirstUnnestInStep,
- boolean isLastUnnestInIntermediateStep) throws AlgebricksException;
+ List<String> fieldName, boolean isFirstArrayStep, boolean isLastUnnestInIntermediateStep)
+ throws AlgebricksException;
void executeActionOnFinalArrayStep(ARecordType startingStepRecordType, List<String> fieldName,
boolean isNonArrayStep, boolean requiresOnlyOneUnnest) throws AlgebricksException;
@@ -352,24 +390,24 @@ public class ArrayIndexUtil {
private static class ArrayPath {
private final List<String> flattenedFieldName;
- private final List<Integer> flattenedDepthIndicators;
+ private final List<Boolean> unnestFlags;
private List<List<String>> fieldNamesPerArray;
- private List<Integer> depthOfArraySteps;
+ private List<Boolean> unnestFlagsPerArray;
- public ArrayPath(List<String> flattenedFieldName, List<Integer> flattenedDepthIndicators) {
+ public ArrayPath(List<String> flattenedFieldName, List<Boolean> unnestFlags) {
this.flattenedFieldName = flattenedFieldName;
- this.flattenedDepthIndicators = flattenedDepthIndicators;
+ this.unnestFlags = unnestFlags;
}
public ArrayPath invoke() {
fieldNamesPerArray = new ArrayList<>();
- depthOfArraySteps = new ArrayList<>();
+ unnestFlagsPerArray = new ArrayList<>();
List<String> workingRecordPath = new ArrayList<>();
- for (int i = 0; i < flattenedDepthIndicators.size(); i++) {
+ for (int i = 0; i < unnestFlags.size(); i++) {
workingRecordPath.add(flattenedFieldName.get(i));
- if (i == flattenedDepthIndicators.size() - 1 || flattenedDepthIndicators.get(i) > 0) {
- depthOfArraySteps.add(flattenedDepthIndicators.get(i));
+ if (i == unnestFlags.size() - 1 || unnestFlags.get(i)) {
+ unnestFlagsPerArray.add(unnestFlags.get(i));
fieldNamesPerArray.add(workingRecordPath);
workingRecordPath = new ArrayList<>();
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java
index b2026d7..6a0c44f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java
@@ -153,9 +153,7 @@ public class KeyFieldTypeUtil {
ARecordType sourceType =
(e.getSourceIndicator() == Index.RECORD_INDICATOR) ? recordType : metaRecordType;
Pair<IAType, Boolean> keyPairType = ArrayIndexUtil.getNonNullableOpenFieldType(e.getTypeList().get(i),
- ArrayIndexUtil.getFlattenedKeyFieldNames(e.getUnnestList(), e.getProjectList().get(i)),
- sourceType,
- ArrayIndexUtil.getArrayDepthIndicator(e.getUnnestList(), e.getProjectList().get(i)));
+ e.getUnnestList(), e.getProjectList().get(i), sourceType);
indexKeyTypes.add(keyPairType.first);
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
index fdb21a2..c1f8c7b 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
@@ -57,7 +57,6 @@ import org.apache.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
import org.apache.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
-import org.apache.hyracks.algebricks.runtime.evaluators.ColumnAccessEvalFactory;
import org.apache.hyracks.algebricks.runtime.operators.aggreg.SimpleAlgebricksAccumulatingAggregatorFactory;
import org.apache.hyracks.algebricks.runtime.operators.base.SinkRuntimeFactory;
import org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
@@ -78,14 +77,13 @@ import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndexOperationsHelper {
private final int numAtomicSecondaryKeys, numArraySecondaryKeys, numTotalSecondaryKeys;
- private final Index.ArrayIndexDetails arrayIndexDetails;
private final EvalFactoryAndRecDescStackBuilder evalFactoryAndRecDescStackBuilder =
new EvalFactoryAndRecDescStackBuilder();
- // TODO (GLENN): Phase these out and use the UNNEST / PROJECT scheme instead.
+ private final Index.ArrayIndexDetails arrayIndexDetails;
private final List<List<String>> flattenedFieldNames;
private final List<IAType> flattenedKeyTypes;
- private final List<List<Integer>> depthIndicators;
+ private final List<List<Boolean>> unnestFlags;
protected SecondaryArrayIndexBTreeOperationsHelper(Dataset dataset, Index index, MetadataProvider metadataProvider,
SourceLocation sourceLoc) throws AlgebricksException {
@@ -94,19 +92,19 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex
flattenedFieldNames = new ArrayList<>();
flattenedKeyTypes = new ArrayList<>();
- depthIndicators = new ArrayList<>();
+ unnestFlags = new ArrayList<>();
for (Index.ArrayIndexElement e : arrayIndexDetails.getElementList()) {
if (e.getUnnestList().isEmpty()) {
flattenedFieldNames.add(e.getProjectList().get(0));
flattenedKeyTypes.add(e.getTypeList().get(0));
- depthIndicators
- .add(ArrayIndexUtil.getArrayDepthIndicator(e.getUnnestList(), e.getProjectList().get(0)));
+ unnestFlags.add(ArrayIndexUtil.getUnnestFlags(e.getUnnestList(), e.getProjectList().get(0)));
+
} else {
for (int i = 0; i < e.getProjectList().size(); i++) {
List<String> project = e.getProjectList().get(i);
flattenedFieldNames.add(ArrayIndexUtil.getFlattenedKeyFieldNames(e.getUnnestList(), project));
- depthIndicators.add(ArrayIndexUtil.getArrayDepthIndicator(e.getUnnestList(), project));
flattenedKeyTypes.add(e.getTypeList().get(i));
+ unnestFlags.add(ArrayIndexUtil.getUnnestFlags(e.getUnnestList(), project));
}
}
}
@@ -127,7 +125,7 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex
numArraySecondaryKeys = numTotalSecondaryKeys - numAtomicSecondaryKeys;
}
- private int findPosOfArrayIndex() throws AsterixException {
+ private int findPosOfArrayIndexElement() throws AsterixException {
for (int i = 0; i < arrayIndexDetails.getElementList().size(); i++) {
if (!arrayIndexDetails.getElementList().get(i).getUnnestList().isEmpty()) {
return i;
@@ -163,9 +161,7 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex
ARecordType sourceType = (e.getSourceIndicator() == 0) ? itemType : metaType;
addSKEvalFactories(isOverridingKeyFieldTypes ? enforcedItemType : sourceType, flattenedListPos, false);
Pair<IAType, Boolean> keyTypePair = ArrayIndexUtil.getNonNullableOpenFieldType(e.getTypeList().get(i),
- ArrayIndexUtil.getFlattenedKeyFieldNames(e.getUnnestList(), e.getProjectList().get(i)),
- sourceType,
- ArrayIndexUtil.getArrayDepthIndicator(e.getUnnestList(), e.getProjectList().get(i)));
+ e.getUnnestList(), e.getProjectList().get(i), sourceType);
IAType keyType = keyTypePair.first;
anySecondaryKeyIsNullable = anySecondaryKeyIsNullable || keyTypePair.second;
ISerializerDeserializer keySerde = serdeProvider.getSerializerDeserializer(keyType);
@@ -242,14 +238,15 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex
return;
}
- List<Integer> arrayDepthIndicators = depthIndicators.get(fieldPos);
- List<String> fieldNames = flattenedFieldNames.get(fieldPos);
- if (arrayDepthIndicators.stream().noneMatch(b -> b > 0)) {
+ List<String> flattenedFieldName = flattenedFieldNames.get(fieldPos);
+ List<Boolean> workingUnnestFlags = unnestFlags.get(fieldPos);
+ if (workingUnnestFlags.stream().noneMatch(b -> b)) {
addAtomicFieldToBuilder(recordType, fieldPos);
+
} else {
EvalFactoryAndRecDescInvoker commandExecutor =
new EvalFactoryAndRecDescInvoker(!evalFactoryAndRecDescStackBuilder.isUnnestEvalPopulated());
- ArrayIndexUtil.walkArrayPath(recordType, fieldNames, arrayDepthIndicators, commandExecutor);
+ ArrayIndexUtil.walkArrayPath(recordType, flattenedFieldName, workingUnnestFlags, commandExecutor);
}
}
@@ -278,13 +275,12 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex
sourceOp = targetOp;
}
- // TODO (GLENN): Refactor to use UNNEST + PROJECT scheme.
// Perform the unnest work.
final Mutable<IOperatorDescriptor> sourceOpRef = new MutableObject<>(sourceOp);
final Mutable<IOperatorDescriptor> targetOpRef = new MutableObject<>(targetOp);
LoadingJobBuilder jobBuilder = new LoadingJobBuilder(spec, sourceOpRef, targetOpRef);
- int posOfArrayIndex = findPosOfArrayIndex();
- ArrayIndexUtil.walkArrayPath(flattenedFieldNames.get(posOfArrayIndex), depthIndicators.get(posOfArrayIndex),
+ int posOfArrayElement = findPosOfArrayIndexElement();
+ ArrayIndexUtil.walkArrayPath(flattenedFieldNames.get(posOfArrayElement), unnestFlags.get(posOfArrayElement),
jobBuilder);
sourceOp = sourceOpRef.getValue();
@@ -450,8 +446,7 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex
: inputWidth + numTotalSecondaryKeys + numFilterFields).toArray();
for (int i = 0; i < numTotalSecondaryKeys; i++) {
int sizeOfFieldNamesForI = flattenedFieldNames.get(i).size();
- if (depthIndicators.get(i).get(sizeOfFieldNamesForI - 1) != 0
- && (depthIndicators.get(i).stream().anyMatch(b -> b > 0))) {
+ if (unnestFlags.get(i).get(sizeOfFieldNamesForI - 1)) {
projectionList[i] = numPrimaryKeys + 1;
} else {
projectionList[i] = outColumns[outColumnsCursor++];
@@ -469,9 +464,9 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex
outColumns = IntStream.range(inputWidth, inputWidth + numArraySecondaryKeys).toArray();
for (int i = 0; i < numTotalSecondaryKeys; i++) {
int sizeOfFieldNamesForI = flattenedFieldNames.get(i).size();
- if (depthIndicators.get(i).stream().noneMatch(b -> b > 0)) {
+ if (unnestFlags.get(i).stream().noneMatch(b -> b)) {
projectionList[i] = numPrimaryKeys + atomicSKCursor++;
- } else if (depthIndicators.get(i).get(sizeOfFieldNamesForI - 1) == 0) {
+ } else if (!unnestFlags.get(i).get(sizeOfFieldNamesForI - 1)) {
projectionList[i] = outColumns[arraySKCursor++];
} else {
projectionList[i] = numPrimaryKeys + numAtomicSecondaryKeys + numFilterFields + 1;
@@ -525,25 +520,20 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex
@Override
public void executeActionOnEachArrayStep(ARecordType startingStepRecordType, IAType workingType,
- List<String> fieldName, boolean isFirstArrayStep, boolean isFirstUnnestInStep,
- boolean isLastUnnestInIntermediateStep) throws AlgebricksException {
+ List<String> fieldName, boolean isFirstArrayStep, boolean isLastUnnestInIntermediateStep)
+ throws AlgebricksException {
if (!this.isFirstWalk) {
// We have already added the appropriate UNNESTs.
return;
}
int sourceColumnForNestedArrays = numPrimaryKeys + numAtomicSecondaryKeys + numFilterFields;
- if (isFirstUnnestInStep) {
- int sourceColumnForFirstUnnestInAtomicPath =
- isFirstArrayStep ? numPrimaryKeys : sourceColumnForNestedArrays;
- IScalarEvaluatorFactory sef = metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
- metadataProvider.getFunctionManager(), startingStepRecordType, fieldName,
- sourceColumnForFirstUnnestInAtomicPath, sourceLoc);
- evalFactoryAndRecDescStackBuilder.addUnnest(sef, workingType);
- } else {
- IScalarEvaluatorFactory sef = new ColumnAccessEvalFactory(sourceColumnForNestedArrays);
- evalFactoryAndRecDescStackBuilder.addUnnest(sef, workingType);
- }
+ int sourceColumnForFirstUnnestInAtomicPath =
+ isFirstArrayStep ? numPrimaryKeys : sourceColumnForNestedArrays;
+ IScalarEvaluatorFactory sef = metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+ metadataProvider.getFunctionManager(), startingStepRecordType, fieldName,
+ sourceColumnForFirstUnnestInAtomicPath, sourceLoc);
+ evalFactoryAndRecDescStackBuilder.addUnnest(sef, workingType);
}
@Override
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
index dd303fc..b1d610f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
@@ -61,8 +61,8 @@ public class TypeUtil {
}
private static class EnforcedTypeBuilder {
- private final Deque<Triple<IAType, String, Integer>> typeStack = new ArrayDeque<>();
- private List<Integer> keyDepthIndicators;
+ private final Deque<Triple<IAType, String, Boolean>> typeStack = new ArrayDeque<>();
+ private List<Boolean> keyUnnestFlags;
private List<String> keyFieldNames;
private ARecordType baseRecordType;
private IAType keyFieldType;
@@ -72,11 +72,11 @@ public class TypeUtil {
private IAType endOfOpenTypeBuild;
private int indexOfOpenPart;
- public void reset(ARecordType baseRecordType, List<String> keyFieldNames, List<Integer> keyDepthIndicators,
+ public void reset(ARecordType baseRecordType, List<String> keyFieldNames, List<Boolean> keyUnnestFlags,
IAType keyFieldType) {
this.baseRecordType = baseRecordType;
this.keyFieldNames = keyFieldNames;
- this.keyDepthIndicators = keyDepthIndicators;
+ this.keyUnnestFlags = keyUnnestFlags;
this.keyFieldType = keyFieldType;
}
@@ -90,21 +90,19 @@ public class TypeUtil {
IAType typeIntermediate = baseRecordType;
List<String> subFieldName = new ArrayList<>();
for (int i = 0; i < keyFieldNames.size() - 1; i++) {
- typeStack.push(new Triple<>(typeIntermediate, keyFieldNames.get(i),
- (i == 0) ? 0 : keyDepthIndicators.get(i - 1)));
+ typeStack.push(
+ new Triple<>(typeIntermediate, keyFieldNames.get(i), i != 0 && keyUnnestFlags.get(i - 1)));
bridgeNameFoundFromOpenTypeBuild = typeIntermediate.getTypeName();
- if (i == 0 || keyDepthIndicators.get(i - 1) == 0) {
+ if (i == 0 || !keyUnnestFlags.get(i - 1)) {
subFieldName.add(keyFieldNames.get(i));
} else {
- // We have a multi-valued intermediate. Traverse the array first, then add our field name.
- for (int j = 0; j < keyDepthIndicators.get(i - 1); j++) {
- typeIntermediate = TypeComputeUtils.extractListItemType(typeIntermediate);
- if (typeIntermediate == null) {
- String fName = String.join(".", subFieldName);
- throw new AsterixException(ErrorCode.COMPILATION_ERROR,
- "Wrong level of array nesting for field: " + fName);
- }
+ // We have a multi-valued intermediate. Perform our UNNEST then add our field name.
+ typeIntermediate = TypeComputeUtils.extractListItemType(typeIntermediate);
+ if (typeIntermediate == null) {
+ String fName = String.join(".", subFieldName);
+ throw new AsterixException(ErrorCode.COMPILATION_ERROR,
+ "No list item type found. Wrong type given from field " + fName);
}
subFieldName.add(keyFieldNames.get(i));
}
@@ -133,27 +131,27 @@ public class TypeUtil {
}
private IAType buildNewForOpenType() {
- int depthOfOpenType = keyDepthIndicators.subList(indexOfOpenPart + 1, keyDepthIndicators.size()).stream()
- .filter(i -> i != 0).findFirst().orElse(0);
- IAType resultant = nestArrayType(keyFieldType, depthOfOpenType);
+ boolean isTypeWithUnnest = keyUnnestFlags.subList(indexOfOpenPart + 1, keyUnnestFlags.size()).stream()
+ .filter(i -> i).findFirst().orElse(false);
+ IAType resultant = nestArrayType(keyFieldType, isTypeWithUnnest);
// Build the type (list or record) that holds the type (list or record) above.
resultant = nestArrayType(
new ARecordType(keyFieldNames.get(keyFieldNames.size() - 2),
new String[] { keyFieldNames.get(keyFieldNames.size() - 1) },
new IAType[] { AUnionType.createUnknownableType(resultant) }, true),
- keyDepthIndicators.get(indexOfOpenPart));
+ keyUnnestFlags.get(indexOfOpenPart));
// Create open part of the nested field.
for (int i = keyFieldNames.size() - 3; i > (indexOfOpenPart - 1); i--) {
resultant = nestArrayType(
new ARecordType(keyFieldNames.get(i), new String[] { keyFieldNames.get(i + 1) },
new IAType[] { AUnionType.createUnknownableType(resultant) }, true),
- keyDepthIndicators.get(i));
+ keyUnnestFlags.get(i));
}
// Now update the parent to include this optional field, accounting for intermediate arrays.
- Triple<IAType, String, Integer> gapTriple = this.typeStack.pop();
+ Triple<IAType, String, Boolean> gapTriple = this.typeStack.pop();
ARecordType parentRecord =
(ARecordType) unnestArrayType(TypeComputeUtils.getActualType(gapTriple.first), gapTriple.third);
IAType[] parentFieldTypes = ArrayUtils.addAll(parentRecord.getFieldTypes().clone(),
@@ -168,9 +166,9 @@ public class TypeUtil {
private IAType buildNewForFullyClosedType() throws AsterixException {
// The schema is closed all the way to the field itself.
IAType typeIntermediate = TypeComputeUtils.getActualType(endOfOpenTypeBuild);
- int depthOfOpenType = (indexOfOpenPart == 0) ? 0 : keyDepthIndicators.get(indexOfOpenPart - 1);
- int depthOfKeyType = keyDepthIndicators.get(indexOfOpenPart);
- ARecordType lastNestedRecord = (ARecordType) unnestArrayType(typeIntermediate, depthOfOpenType);
+ boolean isOpenTypeWithUnnest = indexOfOpenPart != 0 && keyUnnestFlags.get(indexOfOpenPart - 1);
+ boolean isKeyTypeWithUnnest = keyUnnestFlags.get(indexOfOpenPart);
+ ARecordType lastNestedRecord = (ARecordType) unnestArrayType(typeIntermediate, isOpenTypeWithUnnest);
Map<String, IAType> recordNameTypesMap = createRecordNameTypeMap(lastNestedRecord);
// If an enforced field already exists, verify that the type is correct.
@@ -186,21 +184,21 @@ public class TypeUtil {
}
if (enforcedFieldType == null) {
recordNameTypesMap.put(keyFieldNames.get(keyFieldNames.size() - 1),
- AUnionType.createUnknownableType(nestArrayType(keyFieldType, depthOfKeyType)));
+ AUnionType.createUnknownableType(nestArrayType(keyFieldType, isKeyTypeWithUnnest)));
}
// Build the nested record, and account for the wrapping array.
IAType resultant = nestArrayType(
new ARecordType(lastNestedRecord.getTypeName(), recordNameTypesMap.keySet().toArray(new String[0]),
recordNameTypesMap.values().toArray(new IAType[0]), lastNestedRecord.isOpen()),
- depthOfOpenType);
+ isOpenTypeWithUnnest);
return keepUnknown(endOfOpenTypeBuild, resultant);
}
private ARecordType buildRestOfRecord(IAType newTypeToAdd) {
IAType resultant = TypeComputeUtils.getActualType(newTypeToAdd);
while (!typeStack.isEmpty()) {
- Triple<IAType, String, Integer> typeFromStack = typeStack.pop();
+ Triple<IAType, String, Boolean> typeFromStack = typeStack.pop();
IAType typeIntermediate = unnestArrayType(typeFromStack.first, typeFromStack.third);
ARecordType recordType = (ARecordType) typeIntermediate;
IAType[] fieldTypes = recordType.getFieldTypes().clone();
@@ -228,18 +226,13 @@ public class TypeUtil {
return updatedRecordType;
}
- private static IAType nestArrayType(IAType originalType, int depthOfArrays) {
- IAType resultant = originalType;
- for (int i = 0; i < depthOfArrays; i++) {
- resultant =
- new AOrderedListType(resultant, (i == depthOfArrays - 1) ? originalType.getTypeName() : null);
- }
- return resultant;
+ private static IAType nestArrayType(IAType originalType, boolean isWithinArray) {
+ return (isWithinArray) ? new AOrderedListType(originalType, originalType.getTypeName()) : originalType;
}
- private static IAType unnestArrayType(IAType originalType, int depthOfArrays) {
+ private static IAType unnestArrayType(IAType originalType, boolean isWithinArray) {
IAType resultant = originalType;
- for (int i = 0; i < depthOfArrays; i++) {
+ if (isWithinArray) {
resultant = TypeComputeUtils.extractListItemType(resultant);
if (resultant != null) {
resultant = TypeComputeUtils.getActualType(resultant);
@@ -299,7 +292,7 @@ public class TypeUtil {
"Indexing an open field is only supported on the record part");
}
enforcedTypeBuilder.reset(enforcedRecordType, keyFieldNames.get(i),
- Collections.nCopies(keyFieldNames.get(i).size(), 0), keyFieldTypes.get(i));
+ Collections.nCopies(keyFieldNames.get(i).size(), false), keyFieldTypes.get(i));
validateRecord(enforcedRecordType);
enforcedRecordType = enforcedTypeBuilder.build();
}
@@ -319,7 +312,7 @@ public class TypeUtil {
"Indexing an open field is only supported on the record part");
}
enforcedTypeBuilder.reset(enforcedRecordType, keyFieldNames.get(i),
- Collections.nCopies(keyFieldNames.get(i).size(), 0), keyFieldTypes.get(i));
+ Collections.nCopies(keyFieldNames.get(i).size(), false), keyFieldTypes.get(i));
validateRecord(enforcedRecordType);
enforcedRecordType = enforcedTypeBuilder.build();
}
@@ -342,7 +335,7 @@ public class TypeUtil {
List<String> project = projectList.get(i);
enforcedTypeBuilder.reset(enforcedRecordType,
ArrayIndexUtil.getFlattenedKeyFieldNames(unnestList, project),
- ArrayIndexUtil.getArrayDepthIndicator(unnestList, project), typeList.get(i));
+ ArrayIndexUtil.getUnnestFlags(unnestList, project), typeList.get(i));
validateRecord(enforcedRecordType);
enforcedRecordType = enforcedTypeBuilder.build();
}