You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by gg...@apache.org on 2021/07/23 22:53:34 UTC
[asterixdb] branch master updated: [ASTERIXDB-2932][IDX] Fixing
open type building for array indexes.
This is an automated email from the ASF dual-hosted git repository.
ggalvizo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new d80b5de [ASTERIXDB-2932][IDX] Fixing open type building for array indexes.
d80b5de is described below
commit d80b5de2615d74106d17f72bf93a4cdb0439b8a4
Author: ggalvizo <gg...@uci.edu>
AuthorDate: Mon Jul 19 09:41:35 2021 -1000
[ASTERIXDB-2932][IDX] Fixing open type building for array indexes.
- user mode changes: no
- storage format changes: no
- interface changes: no
Fix to allow for more complex nesting structures for array indexes
(partially typed indexes + records that hold subrecords that hold
arrays). This also fixes the open type names to be more consistent
with the closed types.
Change-Id: If6cf7d13081a6f58457ceefed5c090208fedf8a0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12403
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Glenn Galvizo <gg...@uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
---
.../complex-structures-01.1.ddl.sqlpp | 79 ++++++++++++++++++++++
.../complex-structures-01.2.query.sqlpp | 24 +++++++
.../complex-structures/complex-structures-01.1.adm | 8 +++
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 ++
.../apache/asterix/metadata/utils/TypeUtil.java | 51 ++++++++------
5 files changed, 147 insertions(+), 20 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.1.ddl.sqlpp
new file mode 100644
index 0000000..c0a4d1d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.1.ddl.sqlpp
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE TestDataverse IF EXISTS;
+CREATE DATAVERSE TestDataverse;
+USE TestDataverse;
+CREATE TYPE TestType AS {
+ _id: uuid,
+ closed_field_1: {
+ closed_field_2: [{
+ closed_field_3: {
+ closed_field_4: [{
+ closed_field_5: bigint
+ }]
+ }
+ }]
+ }
+};
+CREATE DATASET TestDataset (TestType)
+PRIMARY KEY _id AUTOGENERATED;
+
+
+-- Fully open index 1. Unnest flags: [0, 1, 0]
+CREATE INDEX testIndex1
+ON TestDataset ( UNNEST open_field_1.open_field_2
+ SELECT open_field_3a : bigint );
+
+-- Fully open index 2. Unnest flags: [0, 1, 0, 0]
+CREATE INDEX testIndex2
+ON TestDataset ( UNNEST open_field_1.open_field_2
+ SELECT open_field_3b.open_field_4 : bigint );
+
+-- Fully open index 3. Unnest flags: [0, 1, 0, 1]
+CREATE INDEX testIndex3
+ON TestDataset ( UNNEST open_field_1.open_field_2
+ UNNEST open_field_3c.open_field_4a : bigint );
+
+-- Fully open index 4. Unnest flags: [0, 1, 0, 1, 0]
+CREATE INDEX testIndex4
+ON TestDataset ( UNNEST open_field_1.open_field_2
+ UNNEST open_field_3c.open_field_4b
+ SELECT open_field_5 : bigint );
+
+
+-- Partially open index 1. [0, 1, 0]
+CREATE INDEX testIndex1c
+ON TestDataset ( UNNEST closed_field_1.open_field_2
+ SELECT open_field_3a : bigint );
+
+-- Partially open index 2. Unnest flags: [0, 1, 0, 0]
+CREATE INDEX testIndex2c
+ON TestDataset ( UNNEST closed_field_1.closed_field_2
+ SELECT open_field_3b.open_field_4 : bigint );
+
+-- Partially open index 3. Unnest flags: [0, 1, 0, 1]
+CREATE INDEX testIndex3c
+ON TestDataset ( UNNEST closed_field_1.closed_field_2
+ UNNEST closed_field_3.open_field_4a : bigint );
+
+-- Partially open index 4. Unnest flags: [0, 1, 0, 1, 0]
+CREATE INDEX testIndex4c
+ON TestDataset ( UNNEST closed_field_1.closed_field_2
+ UNNEST closed_field_3.closed_field_4
+ SELECT open_field_5 : bigint );
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.2.query.sqlpp
new file mode 100644
index 0000000..1b854c3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.2.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+FROM Metadata.`Index` D
+WHERE D.IndexName LIKE "testIndex%" AND
+ D.DataverseName = "TestDataverse"
+SELECT D.SearchKey, D.SearchKeyElements, D.SearchKeyType
+ORDER BY D.IndexName ASC;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/metadata/open/complex-structures/complex-structures-01.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/metadata/open/complex-structures/complex-structures-01.1.adm
new file mode 100644
index 0000000..7618b68
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/metadata/open/complex-structures/complex-structures-01.1.adm
@@ -0,0 +1,8 @@
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [ "open_field_1", "open_field_2" ] ], "ProjectList": [ [ "open_field_3a" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [ "closed_field_1", "open_field_2" ] ], "ProjectList": [ [ "open_field_3a" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [ "open_field_1", "open_field_2" ] ], "ProjectList": [ [ "open_field_3b", "open_field_4" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [ "closed_field_1", "closed_field_2" ] ], "ProjectList": [ [ "open_field_3b", "open_field_4" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [ "open_field_1", "open_field_2" ], [ "open_field_3c", "open_field_4a" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [ "closed_field_1", "closed_field_2" ], [ "closed_field_3", "open_field_4a" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [ "open_field_1", "open_field_2" ], [ "open_field_3c", "open_field_4b" ] ], "ProjectList": [ [ "open_field_5" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [ ], "SearchKeyElements": [ { "UnnestList": [ [ "closed_field_1", "closed_field_2" ], [ "closed_field_3", "closed_field_4" ] ], "ProjectList": [ [ "open_field_5" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 74dc47b..03be890 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -8066,6 +8066,11 @@
<output-dir compare="Text">use-case-4</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="array-index/metadata/open">
+ <compilation-unit name="complex-structures">
+ <output-dir compare="Text">complex-structures</output-dir>
+ </compilation-unit>
+ </test-case>
<!-- <test-case FilePath="array-index/metadata/closed">-->
<!-- <compilation-unit name="with-composite-sk">-->
<!-- <output-dir compare="Text">with-composite-sk</output-dir>-->
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
index b1d610f..8a9236e 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
@@ -131,23 +131,26 @@ public class TypeUtil {
}
private IAType buildNewForOpenType() {
- boolean isTypeWithUnnest = keyUnnestFlags.subList(indexOfOpenPart + 1, keyUnnestFlags.size()).stream()
- .filter(i -> i).findFirst().orElse(false);
- IAType resultant = nestArrayType(keyFieldType, isTypeWithUnnest);
-
- // Build the type (list or record) that holds the type (list or record) above.
- resultant = nestArrayType(
- new ARecordType(keyFieldNames.get(keyFieldNames.size() - 2),
- new String[] { keyFieldNames.get(keyFieldNames.size() - 1) },
- new IAType[] { AUnionType.createUnknownableType(resultant) }, true),
- keyUnnestFlags.get(indexOfOpenPart));
-
- // Create open part of the nested field.
- for (int i = keyFieldNames.size() - 3; i > (indexOfOpenPart - 1); i--) {
- resultant = nestArrayType(
- new ARecordType(keyFieldNames.get(i), new String[] { keyFieldNames.get(i + 1) },
- new IAType[] { AUnionType.createUnknownableType(resultant) }, true),
- keyUnnestFlags.get(i));
+ // Walk backwards through our flags and construct the desired type.
+ List<Boolean> unnestFlagsForOpenType = keyUnnestFlags.subList(indexOfOpenPart, keyUnnestFlags.size());
+ List<String> fieldNamesForOpenType = keyFieldNames.subList(indexOfOpenPart, keyFieldNames.size());
+ IAType resultant = keyFieldType;
+ for (int i = unnestFlagsForOpenType.size() - 1; i >= 0; i--) {
+ // Construct the type name.
+ StringBuilder recordTypeNameBuilder = new StringBuilder();
+ recordTypeNameBuilder.append(baseRecordType.getTypeName());
+ for (int j = 0; j < i + indexOfOpenPart; j++) {
+ recordTypeNameBuilder.append("_").append(keyFieldNames.get(j));
+ if (keyUnnestFlags.get(j)) {
+ recordTypeNameBuilder.append("_Item");
+ }
+ }
+
+ // Construct the type itself and account for any array steps.
+ resultant = nestArrayType(resultant, unnestFlagsForOpenType.get(i));
+ resultant =
+ new ARecordType(recordTypeNameBuilder.toString(), new String[] { fieldNamesForOpenType.get(i) },
+ new IAType[] { AUnionType.createUnknownableType(resultant) }, true);
}
// Now update the parent to include this optional field, accounting for intermediate arrays.
@@ -155,9 +158,9 @@ public class TypeUtil {
ARecordType parentRecord =
(ARecordType) unnestArrayType(TypeComputeUtils.getActualType(gapTriple.first), gapTriple.third);
IAType[] parentFieldTypes = ArrayUtils.addAll(parentRecord.getFieldTypes().clone(),
- AUnionType.createUnknownableType(resultant));
+ ((ARecordType) resultant).getFieldTypes().clone());
resultant = new ARecordType(bridgeNameFoundFromOpenTypeBuild,
- ArrayUtils.addAll(parentRecord.getFieldNames(), resultant.getTypeName()), parentFieldTypes, true);
+ ArrayUtils.addAll(parentRecord.getFieldNames(), gapTriple.second), parentFieldTypes, true);
resultant = keepUnknown(gapTriple.first, nestArrayType(resultant, gapTriple.third));
return resultant;
@@ -227,7 +230,15 @@ public class TypeUtil {
}
private static IAType nestArrayType(IAType originalType, boolean isWithinArray) {
- return (isWithinArray) ? new AOrderedListType(originalType, originalType.getTypeName()) : originalType;
+ if (isWithinArray) {
+ String newTypeName = originalType.getTypeName().endsWith("_Item")
+ ? originalType.getTypeName().substring(0, originalType.getTypeName().length() - 5)
+ : originalType.getTypeName();
+ return new AOrderedListType(originalType, newTypeName + "_Array");
+
+ } else {
+ return originalType;
+ }
}
private static IAType unnestArrayType(IAType originalType, boolean isWithinArray) {