You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2021/07/19 19:44:16 UTC

Change in asterixdb[master]: [ASTERIXDB-2932][IDX] Fixing open type building for array indexes.

From Glenn Galvizo <gg...@uci.edu>:

Glenn Galvizo has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12403 )


Change subject: [ASTERIXDB-2932][IDX] Fixing open type building for array indexes.
......................................................................

[ASTERIXDB-2932][IDX] Fixing open type building for array indexes.

- user mode changes: no
- storage format changes: no
- interface changes: no

Fix to allow for more complex nesting structures for array indexes. This
also fixes the open type names to be more consistent with the closed
types.

Change-Id: If6cf7d13081a6f58457ceefed5c090208fedf8a0
---
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.1.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.2.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/metadata/open/complex-structures/complex-structures-01.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
5 files changed, 146 insertions(+), 19 deletions(-)



  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/03/12403/1

diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.1.ddl.sqlpp
new file mode 100644
index 0000000..c0a4d1d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.1.ddl.sqlpp
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE    TestDataverse IF EXISTS;
+CREATE DATAVERSE  TestDataverse;
+USE               TestDataverse;
+CREATE TYPE       TestType AS {
+  _id: uuid,
+  closed_field_1: {
+    closed_field_2: [{
+        closed_field_3: {
+            closed_field_4: [{
+                closed_field_5: bigint
+            }]
+        }
+    }]
+  }
+};
+CREATE DATASET    TestDataset (TestType)
+PRIMARY KEY       _id AUTOGENERATED;
+
+
+-- Fully open index 1. Unnest flags: [0, 1, 0]
+CREATE INDEX      testIndex1
+ON                TestDataset ( UNNEST open_field_1.open_field_2
+                                SELECT open_field_3a : bigint );
+
+-- Fully open index 2. Unnest flags: [0, 1, 0, 0]
+CREATE INDEX      testIndex2
+ON                TestDataset ( UNNEST open_field_1.open_field_2
+                                SELECT open_field_3b.open_field_4 : bigint );
+
+-- Fully open index 3. Unnest flags: [0, 1, 0, 1]
+CREATE INDEX      testIndex3
+ON                TestDataset ( UNNEST open_field_1.open_field_2
+                                UNNEST open_field_3c.open_field_4a : bigint );
+
+-- Fully open index 4. Unnest flags: [0, 1, 0, 1, 0]
+CREATE INDEX      testIndex4
+ON                TestDataset ( UNNEST open_field_1.open_field_2
+                                UNNEST open_field_3c.open_field_4b
+                                SELECT open_field_5 : bigint );
+
+
+-- Partially open index 1.  [0, 1, 0]
+CREATE INDEX      testIndex1c
+ON                TestDataset ( UNNEST closed_field_1.open_field_2
+                                SELECT open_field_3a : bigint );
+
+-- Partially open index 2. Unnest flags: [0, 1, 0, 0]
+CREATE INDEX      testIndex2c
+ON                TestDataset ( UNNEST closed_field_1.closed_field_2
+                                SELECT open_field_3b.open_field_4 : bigint );
+
+-- Partially open index 3. Unnest flags: [0, 1, 0, 1]
+CREATE INDEX      testIndex3c
+ON                TestDataset ( UNNEST closed_field_1.closed_field_2
+                                UNNEST closed_field_3.open_field_4a : bigint );
+
+-- Partially open index 4. Unnest flags: [0, 1, 0, 1, 0]
+CREATE INDEX      testIndex4c
+ON                TestDataset ( UNNEST closed_field_1.closed_field_2
+                                UNNEST closed_field_3.closed_field_4
+                                SELECT open_field_5 : bigint );
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.2.query.sqlpp
new file mode 100644
index 0000000..1b854c3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/metadata/open/complex-structures/complex-structures-01.2.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+FROM      Metadata.`Index` D
+WHERE     D.IndexName LIKE "testIndex%" AND
+          D.DataverseName = "TestDataverse"
+SELECT    D.SearchKey, D.SearchKeyElements, D.SearchKeyType
+ORDER BY  D.IndexName ASC;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/metadata/open/complex-structures/complex-structures-01.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/metadata/open/complex-structures/complex-structures-01.1.adm
new file mode 100644
index 0000000..7618b68
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/metadata/open/complex-structures/complex-structures-01.1.adm
@@ -0,0 +1,8 @@
+{ "SearchKey": [  ], "SearchKeyElements": [ { "UnnestList": [ [ "open_field_1", "open_field_2" ] ], "ProjectList": [ [ "open_field_3a" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [  ], "SearchKeyElements": [ { "UnnestList": [ [ "closed_field_1", "open_field_2" ] ], "ProjectList": [ [ "open_field_3a" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [  ], "SearchKeyElements": [ { "UnnestList": [ [ "open_field_1", "open_field_2" ] ], "ProjectList": [ [ "open_field_3b", "open_field_4" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [  ], "SearchKeyElements": [ { "UnnestList": [ [ "closed_field_1", "closed_field_2" ] ], "ProjectList": [ [ "open_field_3b", "open_field_4" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [  ], "SearchKeyElements": [ { "UnnestList": [ [ "open_field_1", "open_field_2" ], [ "open_field_3c", "open_field_4a" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [  ], "SearchKeyElements": [ { "UnnestList": [ [ "closed_field_1", "closed_field_2" ], [ "closed_field_3", "open_field_4a" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [  ], "SearchKeyElements": [ { "UnnestList": [ [ "open_field_1", "open_field_2" ], [ "open_field_3c", "open_field_4b" ] ], "ProjectList": [ [ "open_field_5" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
+{ "SearchKey": [  ], "SearchKeyElements": [ { "UnnestList": [ [ "closed_field_1", "closed_field_2" ], [ "closed_field_3", "closed_field_4" ] ], "ProjectList": [ [ "open_field_5" ] ] } ], "SearchKeyType": [ [ "int64" ] ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index b9e38a1..7cd2ac2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -8063,6 +8063,11 @@
           <output-dir compare="Text">use-case-4</output-dir>
         </compilation-unit>
       </test-case>
+      <test-case FilePath="array-index/metadata/open">
+        <compilation-unit name="complex-structures">
+          <output-dir compare="Text">complex-structures</output-dir>
+        </compilation-unit>
+      </test-case>
       <!--      <test-case FilePath="array-index/metadata/closed">-->
       <!--        <compilation-unit name="with-composite-sk">-->
       <!--          <output-dir compare="Text">with-composite-sk</output-dir>-->
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
index b1d610f..8a9236e 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java
@@ -131,23 +131,26 @@
         }
 
         private IAType buildNewForOpenType() {
-            boolean isTypeWithUnnest = keyUnnestFlags.subList(indexOfOpenPart + 1, keyUnnestFlags.size()).stream()
-                    .filter(i -> i).findFirst().orElse(false);
-            IAType resultant = nestArrayType(keyFieldType, isTypeWithUnnest);
+            // Walk backwards through our flags and construct the desired type.
+            List<Boolean> unnestFlagsForOpenType = keyUnnestFlags.subList(indexOfOpenPart, keyUnnestFlags.size());
+            List<String> fieldNamesForOpenType = keyFieldNames.subList(indexOfOpenPart, keyFieldNames.size());
+            IAType resultant = keyFieldType;
+            for (int i = unnestFlagsForOpenType.size() - 1; i >= 0; i--) {
+                // Construct the type name.
+                StringBuilder recordTypeNameBuilder = new StringBuilder();
+                recordTypeNameBuilder.append(baseRecordType.getTypeName());
+                for (int j = 0; j < i + indexOfOpenPart; j++) {
+                    recordTypeNameBuilder.append("_").append(keyFieldNames.get(j));
+                    if (keyUnnestFlags.get(j)) {
+                        recordTypeNameBuilder.append("_Item");
+                    }
+                }
 
-            // Build the type (list or record) that holds the type (list or record) above.
-            resultant = nestArrayType(
-                    new ARecordType(keyFieldNames.get(keyFieldNames.size() - 2),
-                            new String[] { keyFieldNames.get(keyFieldNames.size() - 1) },
-                            new IAType[] { AUnionType.createUnknownableType(resultant) }, true),
-                    keyUnnestFlags.get(indexOfOpenPart));
-
-            // Create open part of the nested field.
-            for (int i = keyFieldNames.size() - 3; i > (indexOfOpenPart - 1); i--) {
-                resultant = nestArrayType(
-                        new ARecordType(keyFieldNames.get(i), new String[] { keyFieldNames.get(i + 1) },
-                                new IAType[] { AUnionType.createUnknownableType(resultant) }, true),
-                        keyUnnestFlags.get(i));
+                // Construct the type itself and account for any array steps.
+                resultant = nestArrayType(resultant, unnestFlagsForOpenType.get(i));
+                resultant =
+                        new ARecordType(recordTypeNameBuilder.toString(), new String[] { fieldNamesForOpenType.get(i) },
+                                new IAType[] { AUnionType.createUnknownableType(resultant) }, true);
             }
 
             // Now update the parent to include this optional field, accounting for intermediate arrays.
@@ -155,9 +158,9 @@
             ARecordType parentRecord =
                     (ARecordType) unnestArrayType(TypeComputeUtils.getActualType(gapTriple.first), gapTriple.third);
             IAType[] parentFieldTypes = ArrayUtils.addAll(parentRecord.getFieldTypes().clone(),
-                    AUnionType.createUnknownableType(resultant));
+                    ((ARecordType) resultant).getFieldTypes().clone());
             resultant = new ARecordType(bridgeNameFoundFromOpenTypeBuild,
-                    ArrayUtils.addAll(parentRecord.getFieldNames(), resultant.getTypeName()), parentFieldTypes, true);
+                    ArrayUtils.addAll(parentRecord.getFieldNames(), gapTriple.second), parentFieldTypes, true);
             resultant = keepUnknown(gapTriple.first, nestArrayType(resultant, gapTriple.third));
 
             return resultant;
@@ -227,7 +230,15 @@
         }
 
         private static IAType nestArrayType(IAType originalType, boolean isWithinArray) {
-            return (isWithinArray) ? new AOrderedListType(originalType, originalType.getTypeName()) : originalType;
+            if (isWithinArray) {
+                String newTypeName = originalType.getTypeName().endsWith("_Item")
+                        ? originalType.getTypeName().substring(0, originalType.getTypeName().length() - 5)
+                        : originalType.getTypeName();
+                return new AOrderedListType(originalType, newTypeName + "_Array");
+
+            } else {
+                return originalType;
+            }
         }
 
         private static IAType unnestArrayType(IAType originalType, boolean isWithinArray) {

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12403
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: If6cf7d13081a6f58457ceefed5c090208fedf8a0
Gerrit-Change-Number: 12403
Gerrit-PatchSet: 1
Gerrit-Owner: Glenn Galvizo <gg...@uci.edu>
Gerrit-MessageType: newchange