You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2021/06/30 00:11:20 UTC
[asterixdb] branch master updated (a5c6e93 -> bb9b8f7)
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git.
from a5c6e93 [ASTERIXDB-2830][IDX] Fix INLJ Bug for LOJ Queries w/ Array Indexes
new 11acb53 [NO ISSUE][OTH] Extensible QueryServiceServlet
new 6db8b8e [NO ISSUE] Update maven-antrun-plugin 1.8 -> 3.0.0
new f9cd190 [ASTERIXDB-2817][FUN] Fix IndexOutOfBoundsException in get_object_fields()
new 4b66ee0 [ASTERIXDB-2797][COMP] Fix casting elements of a list constructor
new bba56a5 [ASTERIXDB-2732][FUN] Support missing and union type in get_object_field_value
new d28c0a7 [ASTERIXDB-2804][FUN] Use pseudo-random for random(seed)
new 0a44dbc [NO ISSUE] S3 external datasets: add support to anonymous authentication
new 7b1a724 [NO ISSUE][REP] Account for SSL socket data transfer behavior
new 642fa10 [ASTERIXDB-2857][RT] Incorrect result for nested loop outer join
new 8f21c05 [ASTERIXDB-2905] Error when window function used in subquery
new 4d2a73f [NO ISSUE][EXT]: Retry upon failure for S3 retryable errors
new e577fb0 [ASTERIXDB-2844][COMP] Fix syntax error with UNNEST
new bcde090 [ASTERIXDB-2843][COMP] Fix type computer for scalar aggregates
new 32fcc2a [ASTERIXDB-2886][COMP] Fix RemoveRedundantVariablesRule
new acff469 [NO ISSUE][STO] Close datasets of flushed indexes after recovery
new 294e5ed [NO ISSUE][MD] Ensure metadata node is initialized
new 02a1424 [NO ISSUE][HYR] Expose getParams/getSourceLocation on IFormattedException,Warning
new 7b90dd6 Merge branch 'gerrit/mad-hatter'
new f2639a0 Merge "Merge branch 'gerrit/mad-hatter'" into cheshire-cat
new 562fe20 [ASTERIXDB-2917][COMP] Incorrect result with group by over cross product
new 5daf5ea Merge branch 'gerrit/mad-hatter'
new 9d6c228 Merge branch 'gerrit/cheshire-cat' into 'master'
new bb9b8f7 Merge "Merge branch 'gerrit/cheshire-cat' into 'master'"
The 23 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../optimizer/rules/PushGroupByThroughProduct.java | 17 +++---
.../api/http/server/NCQueryServiceServlet.java | 5 +-
.../api/http/server/QueryServiceServlet.java | 15 ++---
.../message/ExecuteStatementRequestMessage.java | 5 +-
.../asterix/app/translator/QueryTranslator.java | 3 +-
.../gby-cross-join/gby-cross-join.1.ddl.sqlpp | 34 +++++++++++
.../gby-cross-join/gby-cross-join.2.update.sqlpp | 34 +++++++++++
.../gby-cross-join/gby-cross-join.3.query.sqlpp | 25 ++++++++
.../window/misc_01/misc_01.10.query.sqlpp | 34 +++++++++++
.../group-by/gby-cross-join/gby-cross-join.3.adm | 2 +
.../results/window/misc_01/misc_01.10.adm | 4 ++
.../window/misc_01/misc_01.10.ast | 71 ++++++++++++++++++++++
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 ++
.../SqlppWindowAggregationSugarVisitor.java | 7 ++-
asterixdb/pom.xml | 4 +-
15 files changed, 241 insertions(+), 24 deletions(-)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/gby-cross-join/gby-cross-join.1.ddl.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/gby-cross-join/gby-cross-join.2.update.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/gby-cross-join/gby-cross-join.3.query.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/window/misc_01/misc_01.10.query.sqlpp
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/gby-cross-join/gby-cross-join.3.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/window/misc_01/misc_01.10.adm
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/window/misc_01/misc_01.10.ast
[asterixdb] 01/23: [ASTERIXDB-2817][FUN] Fix
IndexOutOfBoundsException in get_object_fields()
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit f9cd190a7fd701a6162030130d009e0ee01710c1
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Thu Jun 3 14:57:55 2021 +0300
[ASTERIXDB-2817][FUN] Fix IndexOutOfBoundsException in get_object_fields()
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Fix IndexOutOfBoundsException in get_object_fields()
- Add test case
- Enable some test case
Change-Id: I36d7915da1193ced12c9185831dc69e3da58517b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11744
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
---
.../queries_sqlpp/objects/ObjectsQueries.xml | 9 ++++--
.../missing-fields/missing-fields.1.ddl.sqlpp | 37 ++++++++++++++++++++++
.../missing-fields/missing-fields.2.update.sqlpp | 26 +++++++++++++++
.../missing-fields/missing-fields.3.query.sqlpp | 24 ++++++++++++++
.../missing-fields/missing-fields.4.query.sqlpp | 23 ++++++++++++++
.../missing-fields/missing-fields.3.adm | 3 ++
.../missing-fields/missing-fields.4.adm | 14 ++++++++
.../tiny-social-example/tiny-social-example.4.adm | 2 +-
.../tiny-social-example/tiny-social-example.5.adm | 20 ++++++------
.../tiny-social-example/tiny-social-example.6.adm | 30 +++++++++---------
.../tiny-social-example/tiny-social-example.7.adm | 8 ++---
.../tiny-social-example/tiny-social-example.8.adm | 24 +++++++-------
.../tiny-social-example/tiny-social-example.9.adm | 12 +++----
.../om/pointables/nonvisitor/ARecordPointable.java | 2 +-
.../functions/records/RecordFieldsUtil.java | 34 ++++++++++----------
15 files changed, 199 insertions(+), 69 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
index e8902b6..a53a15e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
@@ -57,11 +57,11 @@
<output-dir compare="Text">documentation-example</output-dir>
</compilation-unit>
</test-case>
- <!--test-case FilePath="objects/get-object-fields">
+ <test-case FilePath="objects/get-object-fields">
<compilation-unit name="tiny-social-example">
<output-dir compare="Text">tiny-social-example</output-dir>
</compilation-unit>
- </test-case!-->
+ </test-case>
<test-case FilePath="objects/get-object-fields">
<compilation-unit name="tiny-social-example-no-complex-types">
<output-dir compare="Text">tiny-social-example-no-complex-types</output-dir>
@@ -77,6 +77,11 @@
<output-dir compare="Text">tiny-social-example-only-records</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="objects/get-object-fields">
+ <compilation-unit name="missing-fields">
+ <output-dir compare="Text">missing-fields</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="objects/get-object-field-value">
<compilation-unit name="documentation-example">
<output-dir compare="Text">documentation-example</output-dir>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/missing-fields/missing-fields.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/missing-fields/missing-fields.1.ddl.sqlpp
new file mode 100644
index 0000000..4eeae30
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/missing-fields/missing-fields.1.ddl.sqlpp
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing get-object-fields where some schema (complex) fields are missing or null
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE t1 AS {
+ inner_f: string
+};
+
+CREATE TYPE t2 AS {
+ id: int,
+ f: t1?
+};
+
+CREATE DATASET ds1(t2) PRIMARY KEY id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/missing-fields/missing-fields.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/missing-fields/missing-fields.2.update.sqlpp
new file mode 100644
index 0000000..93573bb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/missing-fields/missing-fields.2.update.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+INSERT INTO ds1([
+{"id": 1, "f": null },
+{"id": 2},
+{"id": 3, "f": {"inner_f": "foo", "inner_f2": {"f3": "bar"} } }
+]);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/missing-fields/missing-fields.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/missing-fields/missing-fields.3.query.sqlpp
new file mode 100644
index 0000000..819f46a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/missing-fields/missing-fields.3.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+FROM ds1 AS t
+SELECT VALUE get_object_fields(t)
+ORDER BY t.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/missing-fields/missing-fields.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/missing-fields/missing-fields.4.query.sqlpp
new file mode 100644
index 0000000..d56e1e1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/missing-fields/missing-fields.4.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+FROM `Metadata`.`Dataset` d, get_object_fields(d) f
+WHERE f.`is-open` = false
+SELECT DISTINCT VALUE f.`field-name`
+ORDER BY f.`field-name`;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/missing-fields/missing-fields.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/missing-fields/missing-fields.3.adm
new file mode 100644
index 0000000..ea6f909
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/missing-fields/missing-fields.3.adm
@@ -0,0 +1,3 @@
+[ { "field-name": "id", "field-type": "bigint", "is-open": false }, { "field-name": "f", "field-type": "object", "is-open": false } ]
+[ { "field-name": "id", "field-type": "bigint", "is-open": false }, { "field-name": "f", "field-type": "object", "is-open": false } ]
+[ { "field-name": "id", "field-type": "bigint", "is-open": false }, { "field-name": "f", "field-type": "object", "is-open": false, "nested": [ { "field-name": "inner_f", "field-type": "string", "is-open": false }, { "field-name": "inner_f2", "field-type": "object", "is-open": true, "nested": [ { "field-name": "f3", "field-type": "string", "is-open": true } ] } ] } ]
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/missing-fields/missing-fields.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/missing-fields/missing-fields.4.adm
new file mode 100644
index 0000000..0b364b1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/missing-fields/missing-fields.4.adm
@@ -0,0 +1,14 @@
+"CompactionPolicy"
+"CompactionPolicyProperties"
+"DatasetId"
+"DatasetName"
+"DatasetType"
+"DatatypeDataverseName"
+"DatatypeName"
+"DataverseName"
+"ExternalDetails"
+"GroupName"
+"Hints"
+"InternalDetails"
+"PendingOp"
+"Timestamp"
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.4.adm
index 2938d8c..79f0217 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.4.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.4.adm
@@ -1 +1 @@
-[ { "field-name": "id", "field-type": "INT64", "is-open": false }, { "field-name": "alias", "field-type": "STRING", "is-open": false }, { "field-name": "name", "field-type": "STRING", "is-open": false }, { "field-name": "user-since", "field-type": "DATETIME", "is-open": false }, { "field-name": "friend-ids", "field-type": "UNORDEREDLIST", "is-open": false, "list": [ { "field-type": "INT64" } ] }, { "field-name": "employment", "field-type": "ORDEREDLIST", "is-open": false, "list": [ { "fi [...]
+[ { "field-name": "id", "field-type": "bigint", "is-open": false }, { "field-name": "alias", "field-type": "string", "is-open": false }, { "field-name": "name", "field-type": "string", "is-open": false }, { "field-name": "user-since", "field-type": "datetime", "is-open": false }, { "field-name": "friend-ids", "field-type": "multiset", "is-open": false, "list": [ { "field-type": "bigint" } ] }, { "field-name": "employment", "field-type": "array", "is-open": false, "list": [ { "field-type" [...]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.5.adm
index 74d2d3f..f8e1f55 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.5.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.5.adm
@@ -1,10 +1,10 @@
-[ { "field-name": "id", "field-type": "INT64", "is-open": false }, { "field-name": "alias", "field-type": "STRING", "is-open": false }, { "field-name": "name", "field-type": "STRING", "is-open": false }, { "field-name": "user-since", "field-type": "DATETIME", "is-open": false }, { "field-name": "friend-ids", "field-type": "UNORDEREDLIST", "is-open": false, "list": [ { "field-type": "INT64" }, { "field-type": "INT64" }, { "field-type": "INT64" }, { "field-type": "INT64" } ] }, { "field-na [...]
-[ { "field-name": "id", "field-type": "INT64", "is-open": false }, { "field-name": "alias", "field-type": "STRING", "is-open": false }, { "field-name": "name", "field-type": "STRING", "is-open": false }, { "field-name": "user-since", "field-type": "DATETIME", "is-open": false }, { "field-name": "friend-ids", "field-type": "UNORDEREDLIST", "is-open": false, "list": [ { "field-type": "INT64" }, { "field-type": "INT64" } ] }, { "field-name": "employment", "field-type": "ORDEREDLIST", "is-op [...]
-[ { "field-name": "id", "field-type": "INT64", "is-open": false }, { "field-name": "alias", "field-type": "STRING", "is-open": false }, { "field-name": "name", "field-type": "STRING", "is-open": false }, { "field-name": "user-since", "field-type": "DATETIME", "is-open": false }, { "field-name": "friend-ids", "field-type": "UNORDEREDLIST", "is-open": false, "list": [ { "field-type": "INT64" }, { "field-type": "INT64" }, { "field-type": "INT64" }, { "field-type": "INT64" } ] }, { "field-na [...]
-[ { "field-name": "id", "field-type": "INT64", "is-open": false }, { "field-name": "alias", "field-type": "STRING", "is-open": false }, { "field-name": "name", "field-type": "STRING", "is-open": false }, { "field-name": "user-since", "field-type": "DATETIME", "is-open": false }, { "field-name": "friend-ids", "field-type": "UNORDEREDLIST", "is-open": false, "list": [ { "field-type": "INT64" } ] }, { "field-name": "employment", "field-type": "ORDEREDLIST", "is-open": false, "list": [ { "fi [...]
-[ { "field-name": "id", "field-type": "INT64", "is-open": false }, { "field-name": "alias", "field-type": "STRING", "is-open": false }, { "field-name": "name", "field-type": "STRING", "is-open": false }, { "field-name": "user-since", "field-type": "DATETIME", "is-open": false }, { "field-name": "friend-ids", "field-type": "UNORDEREDLIST", "is-open": false, "list": [ { "field-type": "INT64" }, { "field-type": "INT64" }, { "field-type": "INT64" } ] }, { "field-name": "employment", "field-t [...]
-[ { "field-name": "id", "field-type": "INT64", "is-open": false }, { "field-name": "alias", "field-type": "STRING", "is-open": false }, { "field-name": "name", "field-type": "STRING", "is-open": false }, { "field-name": "user-since", "field-type": "DATETIME", "is-open": false }, { "field-name": "friend-ids", "field-type": "UNORDEREDLIST", "is-open": false, "list": [ { "field-type": "INT64" }, { "field-type": "INT64" }, { "field-type": "INT64" } ] }, { "field-name": "employment", "field-t [...]
-[ { "field-name": "id", "field-type": "INT64", "is-open": false }, { "field-name": "alias", "field-type": "STRING", "is-open": false }, { "field-name": "name", "field-type": "STRING", "is-open": false }, { "field-name": "user-since", "field-type": "DATETIME", "is-open": false }, { "field-name": "friend-ids", "field-type": "UNORDEREDLIST", "is-open": false, "list": [ { "field-type": "INT64" } ] }, { "field-name": "employment", "field-type": "ORDEREDLIST", "is-open": false, "list": [ { "fi [...]
-[ { "field-name": "id", "field-type": "INT64", "is-open": false }, { "field-name": "alias", "field-type": "STRING", "is-open": false }, { "field-name": "name", "field-type": "STRING", "is-open": false }, { "field-name": "user-since", "field-type": "DATETIME", "is-open": false }, { "field-name": "friend-ids", "field-type": "UNORDEREDLIST", "is-open": false, "list": [ { "field-type": "INT64" } ] }, { "field-name": "employment", "field-type": "ORDEREDLIST", "is-open": false, "list": [ { "fi [...]
-[ { "field-name": "id", "field-type": "INT64", "is-open": false }, { "field-name": "alias", "field-type": "STRING", "is-open": false }, { "field-name": "name", "field-type": "STRING", "is-open": false }, { "field-name": "user-since", "field-type": "DATETIME", "is-open": false }, { "field-name": "friend-ids", "field-type": "UNORDEREDLIST", "is-open": false, "list": [ { "field-type": "INT64" }, { "field-type": "INT64" } ] }, { "field-name": "employment", "field-type": "ORDEREDLIST", "is-op [...]
-[ { "field-name": "id", "field-type": "INT64", "is-open": false }, { "field-name": "alias", "field-type": "STRING", "is-open": false }, { "field-name": "name", "field-type": "STRING", "is-open": false }, { "field-name": "user-since", "field-type": "DATETIME", "is-open": false }, { "field-name": "friend-ids", "field-type": "UNORDEREDLIST", "is-open": false, "list": [ { "field-type": "INT64" }, { "field-type": "INT64" }, { "field-type": "INT64" } ] }, { "field-name": "employment", "field-t [...]
+[ { "field-name": "id", "field-type": "bigint", "is-open": false }, { "field-name": "alias", "field-type": "string", "is-open": false }, { "field-name": "name", "field-type": "string", "is-open": false }, { "field-name": "user-since", "field-type": "datetime", "is-open": false }, { "field-name": "friend-ids", "field-type": "multiset", "is-open": false, "list": [ { "field-type": "bigint" }, { "field-type": "bigint" }, { "field-type": "bigint" }, { "field-type": "bigint" } ] }, { "field-na [...]
+[ { "field-name": "id", "field-type": "bigint", "is-open": false }, { "field-name": "alias", "field-type": "string", "is-open": false }, { "field-name": "name", "field-type": "string", "is-open": false }, { "field-name": "user-since", "field-type": "datetime", "is-open": false }, { "field-name": "friend-ids", "field-type": "multiset", "is-open": false, "list": [ { "field-type": "bigint" }, { "field-type": "bigint" } ] }, { "field-name": "employment", "field-type": "array", "is-open": fal [...]
+[ { "field-name": "id", "field-type": "bigint", "is-open": false }, { "field-name": "alias", "field-type": "string", "is-open": false }, { "field-name": "name", "field-type": "string", "is-open": false }, { "field-name": "user-since", "field-type": "datetime", "is-open": false }, { "field-name": "friend-ids", "field-type": "multiset", "is-open": false, "list": [ { "field-type": "bigint" }, { "field-type": "bigint" }, { "field-type": "bigint" }, { "field-type": "bigint" } ] }, { "field-na [...]
+[ { "field-name": "id", "field-type": "bigint", "is-open": false }, { "field-name": "alias", "field-type": "string", "is-open": false }, { "field-name": "name", "field-type": "string", "is-open": false }, { "field-name": "user-since", "field-type": "datetime", "is-open": false }, { "field-name": "friend-ids", "field-type": "multiset", "is-open": false, "list": [ { "field-type": "bigint" } ] }, { "field-name": "employment", "field-type": "array", "is-open": false, "list": [ { "field-type" [...]
+[ { "field-name": "id", "field-type": "bigint", "is-open": false }, { "field-name": "alias", "field-type": "string", "is-open": false }, { "field-name": "name", "field-type": "string", "is-open": false }, { "field-name": "user-since", "field-type": "datetime", "is-open": false }, { "field-name": "friend-ids", "field-type": "multiset", "is-open": false, "list": [ { "field-type": "bigint" }, { "field-type": "bigint" }, { "field-type": "bigint" } ] }, { "field-name": "employment", "field-ty [...]
+[ { "field-name": "id", "field-type": "bigint", "is-open": false }, { "field-name": "alias", "field-type": "string", "is-open": false }, { "field-name": "name", "field-type": "string", "is-open": false }, { "field-name": "user-since", "field-type": "datetime", "is-open": false }, { "field-name": "friend-ids", "field-type": "multiset", "is-open": false, "list": [ { "field-type": "bigint" }, { "field-type": "bigint" }, { "field-type": "bigint" } ] }, { "field-name": "employment", "field-ty [...]
+[ { "field-name": "id", "field-type": "bigint", "is-open": false }, { "field-name": "alias", "field-type": "string", "is-open": false }, { "field-name": "name", "field-type": "string", "is-open": false }, { "field-name": "user-since", "field-type": "datetime", "is-open": false }, { "field-name": "friend-ids", "field-type": "multiset", "is-open": false, "list": [ { "field-type": "bigint" } ] }, { "field-name": "employment", "field-type": "array", "is-open": false, "list": [ { "field-type" [...]
+[ { "field-name": "id", "field-type": "bigint", "is-open": false }, { "field-name": "alias", "field-type": "string", "is-open": false }, { "field-name": "name", "field-type": "string", "is-open": false }, { "field-name": "user-since", "field-type": "datetime", "is-open": false }, { "field-name": "friend-ids", "field-type": "multiset", "is-open": false, "list": [ { "field-type": "bigint" } ] }, { "field-name": "employment", "field-type": "array", "is-open": false, "list": [ { "field-type" [...]
+[ { "field-name": "id", "field-type": "bigint", "is-open": false }, { "field-name": "alias", "field-type": "string", "is-open": false }, { "field-name": "name", "field-type": "string", "is-open": false }, { "field-name": "user-since", "field-type": "datetime", "is-open": false }, { "field-name": "friend-ids", "field-type": "multiset", "is-open": false, "list": [ { "field-type": "bigint" }, { "field-type": "bigint" } ] }, { "field-name": "employment", "field-type": "array", "is-open": fal [...]
+[ { "field-name": "id", "field-type": "bigint", "is-open": false }, { "field-name": "alias", "field-type": "string", "is-open": false }, { "field-name": "name", "field-type": "string", "is-open": false }, { "field-name": "user-since", "field-type": "datetime", "is-open": false }, { "field-name": "friend-ids", "field-type": "multiset", "is-open": false, "list": [ { "field-type": "bigint" }, { "field-type": "bigint" }, { "field-type": "bigint" } ] }, { "field-name": "employment", "field-ty [...]
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.6.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.6.adm
index 73bfe63..9a60421 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.6.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.6.adm
@@ -1,15 +1,15 @@
-[ { "field-name": "message-id", "field-type": "INT64", "is-open": false }, { "field-name": "author-id", "field-type": "INT64", "is-open": false }, { "field-name": "in-response-to", "field-type": "INT64", "is-open": false }, { "field-name": "sender-location", "field-type": "POINT", "is-open": false }, { "field-name": "message", "field-type": "STRING", "is-open": false } ]
-[ { "field-name": "message-id", "field-type": "INT64", "is-open": false }, { "field-name": "author-id", "field-type": "INT64", "is-open": false }, { "field-name": "in-response-to", "field-type": "INT64", "is-open": false }, { "field-name": "sender-location", "field-type": "POINT", "is-open": false }, { "field-name": "message", "field-type": "STRING", "is-open": false } ]
-[ { "field-name": "message-id", "field-type": "INT64", "is-open": false }, { "field-name": "author-id", "field-type": "INT64", "is-open": false }, { "field-name": "in-response-to", "field-type": "INT64", "is-open": false }, { "field-name": "sender-location", "field-type": "POINT", "is-open": false }, { "field-name": "message", "field-type": "STRING", "is-open": false } ]
-[ { "field-name": "message-id", "field-type": "INT64", "is-open": false }, { "field-name": "author-id", "field-type": "INT64", "is-open": false }, { "field-name": "in-response-to", "field-type": "INT64", "is-open": false }, { "field-name": "sender-location", "field-type": "POINT", "is-open": false }, { "field-name": "message", "field-type": "STRING", "is-open": false } ]
-[ { "field-name": "message-id", "field-type": "INT64", "is-open": false }, { "field-name": "author-id", "field-type": "INT64", "is-open": false }, { "field-name": "in-response-to", "field-type": "INT64", "is-open": false }, { "field-name": "sender-location", "field-type": "POINT", "is-open": false }, { "field-name": "message", "field-type": "STRING", "is-open": false } ]
-[ { "field-name": "message-id", "field-type": "INT64", "is-open": false }, { "field-name": "author-id", "field-type": "INT64", "is-open": false }, { "field-name": "in-response-to", "field-type": "INT64", "is-open": false }, { "field-name": "sender-location", "field-type": "POINT", "is-open": false }, { "field-name": "message", "field-type": "STRING", "is-open": false } ]
-[ { "field-name": "message-id", "field-type": "INT64", "is-open": false }, { "field-name": "author-id", "field-type": "INT64", "is-open": false }, { "field-name": "in-response-to", "field-type": "INT64", "is-open": false }, { "field-name": "sender-location", "field-type": "POINT", "is-open": false }, { "field-name": "message", "field-type": "STRING", "is-open": false } ]
-[ { "field-name": "message-id", "field-type": "INT64", "is-open": false }, { "field-name": "author-id", "field-type": "INT64", "is-open": false }, { "field-name": "in-response-to", "field-type": "INT64", "is-open": false }, { "field-name": "sender-location", "field-type": "POINT", "is-open": false }, { "field-name": "message", "field-type": "STRING", "is-open": false } ]
-[ { "field-name": "message-id", "field-type": "INT64", "is-open": false }, { "field-name": "author-id", "field-type": "INT64", "is-open": false }, { "field-name": "in-response-to", "field-type": "INT64", "is-open": false }, { "field-name": "sender-location", "field-type": "POINT", "is-open": false }, { "field-name": "message", "field-type": "STRING", "is-open": false } ]
-[ { "field-name": "message-id", "field-type": "INT64", "is-open": false }, { "field-name": "author-id", "field-type": "INT64", "is-open": false }, { "field-name": "in-response-to", "field-type": "INT64", "is-open": false }, { "field-name": "sender-location", "field-type": "POINT", "is-open": false }, { "field-name": "message", "field-type": "STRING", "is-open": false } ]
-[ { "field-name": "message-id", "field-type": "INT64", "is-open": false }, { "field-name": "author-id", "field-type": "INT64", "is-open": false }, { "field-name": "in-response-to", "field-type": "INT64", "is-open": false }, { "field-name": "sender-location", "field-type": "POINT", "is-open": false }, { "field-name": "message", "field-type": "STRING", "is-open": false } ]
-[ { "field-name": "message-id", "field-type": "INT64", "is-open": false }, { "field-name": "author-id", "field-type": "INT64", "is-open": false }, { "field-name": "in-response-to", "field-type": "INT64", "is-open": false }, { "field-name": "sender-location", "field-type": "POINT", "is-open": false }, { "field-name": "message", "field-type": "STRING", "is-open": false } ]
-[ { "field-name": "message-id", "field-type": "INT64", "is-open": false }, { "field-name": "author-id", "field-type": "INT64", "is-open": false }, { "field-name": "in-response-to", "field-type": "INT64", "is-open": false }, { "field-name": "sender-location", "field-type": "POINT", "is-open": false }, { "field-name": "message", "field-type": "STRING", "is-open": false } ]
-[ { "field-name": "message-id", "field-type": "INT64", "is-open": false }, { "field-name": "author-id", "field-type": "INT64", "is-open": false }, { "field-name": "in-response-to", "field-type": "INT64", "is-open": false }, { "field-name": "sender-location", "field-type": "POINT", "is-open": false }, { "field-name": "message", "field-type": "STRING", "is-open": false } ]
-[ { "field-name": "message-id", "field-type": "INT64", "is-open": false }, { "field-name": "author-id", "field-type": "INT64", "is-open": false }, { "field-name": "in-response-to", "field-type": "INT64", "is-open": false }, { "field-name": "sender-location", "field-type": "POINT", "is-open": false }, { "field-name": "message", "field-type": "STRING", "is-open": false } ]
+[ { "field-name": "message-id", "field-type": "bigint", "is-open": false }, { "field-name": "author-id", "field-type": "bigint", "is-open": false }, { "field-name": "in-response-to", "field-type": "bigint", "is-open": false }, { "field-name": "sender-location", "field-type": "point", "is-open": false }, { "field-name": "message", "field-type": "string", "is-open": false } ]
+[ { "field-name": "message-id", "field-type": "bigint", "is-open": false }, { "field-name": "author-id", "field-type": "bigint", "is-open": false }, { "field-name": "in-response-to", "field-type": "bigint", "is-open": false }, { "field-name": "sender-location", "field-type": "point", "is-open": false }, { "field-name": "message", "field-type": "string", "is-open": false } ]
+[ { "field-name": "message-id", "field-type": "bigint", "is-open": false }, { "field-name": "author-id", "field-type": "bigint", "is-open": false }, { "field-name": "in-response-to", "field-type": "bigint", "is-open": false }, { "field-name": "sender-location", "field-type": "point", "is-open": false }, { "field-name": "message", "field-type": "string", "is-open": false } ]
+[ { "field-name": "message-id", "field-type": "bigint", "is-open": false }, { "field-name": "author-id", "field-type": "bigint", "is-open": false }, { "field-name": "in-response-to", "field-type": "bigint", "is-open": false }, { "field-name": "sender-location", "field-type": "point", "is-open": false }, { "field-name": "message", "field-type": "string", "is-open": false } ]
+[ { "field-name": "message-id", "field-type": "bigint", "is-open": false }, { "field-name": "author-id", "field-type": "bigint", "is-open": false }, { "field-name": "in-response-to", "field-type": "bigint", "is-open": false }, { "field-name": "sender-location", "field-type": "point", "is-open": false }, { "field-name": "message", "field-type": "string", "is-open": false } ]
+[ { "field-name": "message-id", "field-type": "bigint", "is-open": false }, { "field-name": "author-id", "field-type": "bigint", "is-open": false }, { "field-name": "in-response-to", "field-type": "bigint", "is-open": false }, { "field-name": "sender-location", "field-type": "point", "is-open": false }, { "field-name": "message", "field-type": "string", "is-open": false } ]
+[ { "field-name": "message-id", "field-type": "bigint", "is-open": false }, { "field-name": "author-id", "field-type": "bigint", "is-open": false }, { "field-name": "in-response-to", "field-type": "bigint", "is-open": false }, { "field-name": "sender-location", "field-type": "point", "is-open": false }, { "field-name": "message", "field-type": "string", "is-open": false } ]
+[ { "field-name": "message-id", "field-type": "bigint", "is-open": false }, { "field-name": "author-id", "field-type": "bigint", "is-open": false }, { "field-name": "in-response-to", "field-type": "bigint", "is-open": false }, { "field-name": "sender-location", "field-type": "point", "is-open": false }, { "field-name": "message", "field-type": "string", "is-open": false } ]
+[ { "field-name": "message-id", "field-type": "bigint", "is-open": false }, { "field-name": "author-id", "field-type": "bigint", "is-open": false }, { "field-name": "in-response-to", "field-type": "bigint", "is-open": false }, { "field-name": "sender-location", "field-type": "point", "is-open": false }, { "field-name": "message", "field-type": "string", "is-open": false } ]
+[ { "field-name": "message-id", "field-type": "bigint", "is-open": false }, { "field-name": "author-id", "field-type": "bigint", "is-open": false }, { "field-name": "in-response-to", "field-type": "bigint", "is-open": false }, { "field-name": "sender-location", "field-type": "point", "is-open": false }, { "field-name": "message", "field-type": "string", "is-open": false } ]
+[ { "field-name": "message-id", "field-type": "bigint", "is-open": false }, { "field-name": "author-id", "field-type": "bigint", "is-open": false }, { "field-name": "in-response-to", "field-type": "bigint", "is-open": false }, { "field-name": "sender-location", "field-type": "point", "is-open": false }, { "field-name": "message", "field-type": "string", "is-open": false } ]
+[ { "field-name": "message-id", "field-type": "bigint", "is-open": false }, { "field-name": "author-id", "field-type": "bigint", "is-open": false }, { "field-name": "in-response-to", "field-type": "bigint", "is-open": false }, { "field-name": "sender-location", "field-type": "point", "is-open": false }, { "field-name": "message", "field-type": "string", "is-open": false } ]
+[ { "field-name": "message-id", "field-type": "bigint", "is-open": false }, { "field-name": "author-id", "field-type": "bigint", "is-open": false }, { "field-name": "in-response-to", "field-type": "bigint", "is-open": false }, { "field-name": "sender-location", "field-type": "point", "is-open": false }, { "field-name": "message", "field-type": "string", "is-open": false } ]
+[ { "field-name": "message-id", "field-type": "bigint", "is-open": false }, { "field-name": "author-id", "field-type": "bigint", "is-open": false }, { "field-name": "in-response-to", "field-type": "bigint", "is-open": false }, { "field-name": "sender-location", "field-type": "point", "is-open": false }, { "field-name": "message", "field-type": "string", "is-open": false } ]
+[ { "field-name": "message-id", "field-type": "bigint", "is-open": false }, { "field-name": "author-id", "field-type": "bigint", "is-open": false }, { "field-name": "in-response-to", "field-type": "bigint", "is-open": false }, { "field-name": "sender-location", "field-type": "point", "is-open": false }, { "field-name": "message", "field-type": "string", "is-open": false } ]
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.7.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.7.adm
index 429d522..d63f06f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.7.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.7.adm
@@ -1,4 +1,4 @@
-[ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING", "is-open": true }, { "field-name": "followers_count", "field-type": "INT64", "is-open": true } ]
-[ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING", "is-open": true }, { "field-name": "followers_count", "field-type": "INT64", "is-open": true } ]
-[ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING", "is-open": true }, { "field-name": "followers_count", "field-type": "INT64", "is-open": true } ]
-[ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING", "is-open": true }, { "field-name": "followers_count", "field-type": "INT64", "is-open": true } ]
+[ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "string", "is-open": true }, { "field-name": "followers_count", "field-type": "bigint", "is-open": true } ]
+[ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "string", "is-open": true }, { "field-name": "followers_count", "field-type": "bigint", "is-open": true } ]
+[ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "string", "is-open": true }, { "field-name": "followers_count", "field-type": "bigint", "is-open": true } ]
+[ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "string", "is-open": true }, { "field-name": "followers_count", "field-type": "bigint", "is-open": true } ]
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.8.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.8.adm
index c31a69b..e84c545 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.8.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.8.adm
@@ -1,12 +1,12 @@
-[ { "field-name": "tweetid", "field-type": "STRING", "is-open": false }, { "field-name": "user", "field-type": "RECORD", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING [...]
-[ { "field-name": "tweetid", "field-type": "STRING", "is-open": false }, { "field-name": "user", "field-type": "RECORD", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING [...]
-[ { "field-name": "tweetid", "field-type": "STRING", "is-open": false }, { "field-name": "user", "field-type": "RECORD", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING [...]
-[ { "field-name": "tweetid", "field-type": "STRING", "is-open": false }, { "field-name": "user", "field-type": "RECORD", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING [...]
-[ { "field-name": "tweetid", "field-type": "STRING", "is-open": false }, { "field-name": "user", "field-type": "RECORD", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING [...]
-[ { "field-name": "tweetid", "field-type": "STRING", "is-open": false }, { "field-name": "user", "field-type": "RECORD", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING [...]
-[ { "field-name": "tweetid", "field-type": "STRING", "is-open": false }, { "field-name": "user", "field-type": "RECORD", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING [...]
-[ { "field-name": "tweetid", "field-type": "STRING", "is-open": false }, { "field-name": "user", "field-type": "RECORD", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING [...]
-[ { "field-name": "tweetid", "field-type": "STRING", "is-open": false }, { "field-name": "user", "field-type": "RECORD", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING [...]
-[ { "field-name": "tweetid", "field-type": "STRING", "is-open": false }, { "field-name": "user", "field-type": "RECORD", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING [...]
-[ { "field-name": "tweetid", "field-type": "STRING", "is-open": false }, { "field-name": "user", "field-type": "RECORD", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING [...]
-[ { "field-name": "tweetid", "field-type": "STRING", "is-open": false }, { "field-name": "user", "field-type": "RECORD", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "STRING", "is-open": false }, { "field-name": "lang", "field-type": "STRING", "is-open": false }, { "field-name": "friends_count", "field-type": "INT64", "is-open": false }, { "field-name": "statuses_count", "field-type": "INT64", "is-open": false }, { "field-name": "name", "field-type": "STRING [...]
+[ { "field-name": "tweetid", "field-type": "string", "is-open": false }, { "field-name": "user", "field-type": "object", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "stri [...]
+[ { "field-name": "tweetid", "field-type": "string", "is-open": false }, { "field-name": "user", "field-type": "object", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "stri [...]
+[ { "field-name": "tweetid", "field-type": "string", "is-open": false }, { "field-name": "user", "field-type": "object", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "stri [...]
+[ { "field-name": "tweetid", "field-type": "string", "is-open": false }, { "field-name": "user", "field-type": "object", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "stri [...]
+[ { "field-name": "tweetid", "field-type": "string", "is-open": false }, { "field-name": "user", "field-type": "object", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "stri [...]
+[ { "field-name": "tweetid", "field-type": "string", "is-open": false }, { "field-name": "user", "field-type": "object", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "stri [...]
+[ { "field-name": "tweetid", "field-type": "string", "is-open": false }, { "field-name": "user", "field-type": "object", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "stri [...]
+[ { "field-name": "tweetid", "field-type": "string", "is-open": false }, { "field-name": "user", "field-type": "object", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "stri [...]
+[ { "field-name": "tweetid", "field-type": "string", "is-open": false }, { "field-name": "user", "field-type": "object", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "stri [...]
+[ { "field-name": "tweetid", "field-type": "string", "is-open": false }, { "field-name": "user", "field-type": "object", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "stri [...]
+[ { "field-name": "tweetid", "field-type": "string", "is-open": false }, { "field-name": "user", "field-type": "object", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "stri [...]
+[ { "field-name": "tweetid", "field-type": "string", "is-open": false }, { "field-name": "user", "field-type": "object", "is-open": false, "nested": [ { "field-name": "screen-name", "field-type": "string", "is-open": false }, { "field-name": "lang", "field-type": "string", "is-open": false }, { "field-name": "friends_count", "field-type": "bigint", "is-open": false }, { "field-name": "statuses_count", "field-type": "bigint", "is-open": false }, { "field-name": "name", "field-type": "stri [...]
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.9.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.9.adm
index 8693920..967135f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.9.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/get-object-fields/tiny-social-example/tiny-social-example.9.adm
@@ -1,6 +1,6 @@
-{ "count": 12, "field-name": "message-text", "field-type": "STRING" }
-{ "count": 12, "field-name": "referred-topics", "field-type": "UNORDEREDLIST" }
-{ "count": 12, "field-name": "send-time", "field-type": "DATETIME" }
-{ "count": 12, "field-name": "sender-location", "field-type": "POINT" }
-{ "count": 12, "field-name": "tweetid", "field-type": "STRING" }
-{ "count": 12, "field-name": "user", "field-type": "RECORD" }
+{ "count": 12, "field-name": "message-text", "field-type": "string" }
+{ "count": 12, "field-name": "referred-topics", "field-type": "multiset" }
+{ "count": 12, "field-name": "send-time", "field-type": "datetime" }
+{ "count": 12, "field-name": "sender-location", "field-type": "point" }
+{ "count": 12, "field-name": "tweetid", "field-type": "string" }
+{ "count": 12, "field-name": "user", "field-type": "object" }
\ No newline at end of file
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/ARecordPointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/ARecordPointable.java
index af8d5e6..cc3816c 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/ARecordPointable.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/nonvisitor/ARecordPointable.java
@@ -152,7 +152,7 @@ public class ARecordPointable extends AbstractPointable {
&& RecordUtil.isNull(bytes[getNullBitmapOffset(recordType) + fieldId / 4], fieldId);
}
- private boolean isClosedFieldMissing(ARecordType recordType, int fieldId) {
+ public boolean isClosedFieldMissing(ARecordType recordType, int fieldId) {
return getNullBitmapSize(recordType) > 0
&& RecordUtil.isMissing(bytes[getNullBitmapOffset(recordType) + fieldId / 4], fieldId);
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java
index 0d85807..25dd73e 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordFieldsUtil.java
@@ -58,16 +58,15 @@ public class RecordFieldsUtil {
private final static AString nestedName = new AString("nested");
private final static AString listName = new AString("list");
- private IObjectPool<IARecordBuilder, ATypeTag> recordBuilderPool =
- new ListObjectPool<IARecordBuilder, ATypeTag>(new RecordBuilderFactory());
- private IObjectPool<IAsterixListBuilder, ATypeTag> listBuilderPool =
- new ListObjectPool<IAsterixListBuilder, ATypeTag>(new ListBuilderFactory());
- private IObjectPool<IMutableValueStorage, ATypeTag> abvsBuilderPool =
- new ListObjectPool<IMutableValueStorage, ATypeTag>(new AbvsBuilderFactory());
- private IObjectPool<IPointable, ATypeTag> recordPointablePool =
- new ListObjectPool<IPointable, ATypeTag>(ARecordPointable.ALLOCATOR);
- private IObjectPool<IPointable, ATypeTag> listPointablePool =
- new ListObjectPool<IPointable, ATypeTag>(AListPointable.ALLOCATOR);
+ private final IObjectPool<IARecordBuilder, ATypeTag> recordBuilderPool =
+ new ListObjectPool<>(new RecordBuilderFactory());
+ private final IObjectPool<IAsterixListBuilder, ATypeTag> listBuilderPool =
+ new ListObjectPool<>(new ListBuilderFactory());
+ private final IObjectPool<IMutableValueStorage, ATypeTag> abvsBuilderPool =
+ new ListObjectPool<>(new AbvsBuilderFactory());
+ private final IObjectPool<IPointable, ATypeTag> recordPointablePool =
+ new ListObjectPool<>(ARecordPointable.ALLOCATOR);
+ private final IObjectPool<IPointable, ATypeTag> listPointablePool = new ListObjectPool<>(AListPointable.ALLOCATOR);
private final static AOrderedListType listType = new AOrderedListType(BuiltinType.ANY, "fields");
//Better not be a static object.
@@ -114,15 +113,14 @@ public class RecordFieldsUtil {
// write nested or list types
if (tag == ATypeTag.SERIALIZED_RECORD_TYPE_TAG || tag == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
|| tag == ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
- if (!recordAccessor.isClosedFieldNull(recType, i)) {
+ if (!recordAccessor.isClosedFieldNull(recType, i) && !recordAccessor.isClosedFieldMissing(recType, i)) {
IAType fieldType = recordAccessor.getClosedFieldType(recType, i);
ArrayBackedValueStorage tmpValue = getTempBuffer();
tmpValue.reset();
recordAccessor.getClosedFieldValue(recType, i, tmpValue.getDataOutput());
if (tag == ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
addNestedField(tmpValue, fieldType, fieldRecordBuilder, level + 1);
- } else if (tag == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
- || tag == ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
+ } else {
addListField(tmpValue, fieldType, fieldRecordBuilder, level + 1);
}
}
@@ -159,8 +157,7 @@ public class RecordFieldsUtil {
recordAccessor.getOpenFieldValue(recType, i, tmpValue.getDataOutput());
if (tag == ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
addNestedField(tmpValue, fieldType, fieldRecordBuilder, level + 1);
- } else if (tag == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
- || tag == ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
+ } else {
addListField(tmpValue, fieldType, fieldRecordBuilder, level + 1);
}
}
@@ -256,7 +253,7 @@ public class RecordFieldsUtil {
ArrayBackedValueStorage itemValue = getTempBuffer();
IARecordBuilder listRecordBuilder = getRecordBuilder();
- AListPointable list = getListPointable();
+ AListPointable list = getListPointable(fieldType.getTypeTag());
list.set(listArg);
OrderedListBuilder innerListBuilder = getOrderedListBuilder();
@@ -274,6 +271,7 @@ public class RecordFieldsUtil {
if (tagId == ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
ArrayBackedValueStorage tmpAbvs = getTempBuffer();
+ tmpAbvs.reset();
list.getItemValue(act, l, tmpAbvs.getDataOutput());
addNestedField(tmpAbvs, act.getItemType(), listRecordBuilder, level + 1);
}
@@ -288,8 +286,8 @@ public class RecordFieldsUtil {
return (ARecordPointable) recordPointablePool.allocate(ATypeTag.OBJECT);
}
- private AListPointable getListPointable() {
- return (AListPointable) listPointablePool.allocate(ATypeTag.ARRAY);
+ private AListPointable getListPointable(ATypeTag tag) {
+ return (AListPointable) listPointablePool.allocate(tag);
}
private IARecordBuilder getRecordBuilder() {
[asterixdb] 20/23: [ASTERIXDB-2917][COMP] Incorrect result with
group by over cross product
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 562fe20fede366b85f1507df9a962b2202248df6
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Mon Jun 28 11:03:21 2021 -0700
[ASTERIXDB-2917][COMP] Incorrect result with group by over cross product
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Fix incorrect result caused PushGroupByThroughProduct rule
- Add testcase
Change-Id: I85400289d3096c4da38ed73c7576e7c8b79a4909
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12104
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../optimizer/rules/PushGroupByThroughProduct.java | 17 ++++++-----
.../gby-cross-join/gby-cross-join.1.ddl.sqlpp | 34 ++++++++++++++++++++++
.../gby-cross-join/gby-cross-join.2.update.sqlpp | 34 ++++++++++++++++++++++
.../gby-cross-join/gby-cross-join.3.query.sqlpp | 25 ++++++++++++++++
.../group-by/gby-cross-join/gby-cross-join.3.adm | 2 ++
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 ++++
6 files changed, 110 insertions(+), 7 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushGroupByThroughProduct.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushGroupByThroughProduct.java
index 48c4324..6d92b51 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushGroupByThroughProduct.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushGroupByThroughProduct.java
@@ -84,24 +84,27 @@ public class PushGroupByThroughProduct implements IAlgebraicRewriteRule {
Mutable<ILogicalOperator> opLeftRef = join.getInputs().get(0);
ILogicalOperator opLeft = opLeftRef.getValue();
+ Mutable<ILogicalOperator> opRightRef = join.getInputs().get(1);
+ ILogicalOperator opRight = opRightRef.getValue();
switch (canPushThrough(gby, opLeft, decorToPush, decorNotToPush)) {
case REPEATED_DECORS: {
return false;
}
case TRUE: {
- push(opRef, opRef2, 0, decorToPush, decorNotToPush, context);
- return true;
+ if (OperatorPropertiesUtil.isCardinalityZeroOrOne(opRight)) {
+ push(opRef, opRef2, 0, decorToPush, decorNotToPush, context);
+ return true;
+ }
+ return false;
}
case FALSE: {
decorToPush.clear();
- Mutable<ILogicalOperator> opRightRef = join.getInputs().get(1);
- ILogicalOperator opRight = opRightRef.getValue();
- if (canPushThrough(gby, opRight, decorToPush, decorNotToPush) == PushTestResult.TRUE) {
+ if (canPushThrough(gby, opRight, decorToPush, decorNotToPush) == PushTestResult.TRUE
+ && OperatorPropertiesUtil.isCardinalityZeroOrOne(opLeft)) {
push(opRef, opRef2, 1, decorToPush, decorNotToPush, context);
return true;
- } else {
- return false;
}
+ return false;
}
default: {
throw new IllegalStateException();
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/gby-cross-join/gby-cross-join.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/gby-cross-join/gby-cross-join.1.ddl.sqlpp
new file mode 100644
index 0000000..10f5546
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/gby-cross-join/gby-cross-join.1.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Testing group by over cross product (ASTERIXDB-2917)
+ * Expected : SUCCESS
+ */
+
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create dataset t1(id integer not unknown) open type primary key id;
+
+create dataset t2(id integer not unknown) open type primary key id;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/gby-cross-join/gby-cross-join.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/gby-cross-join/gby-cross-join.2.update.sqlpp
new file mode 100644
index 0000000..7fe3777
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/gby-cross-join/gby-cross-join.2.update.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+insert into t1 ([
+ { "id": 1, "x": "a" },
+ { "id": 2, "x": "a" },
+ { "id": 3, "x": "a" },
+ { "id": 4, "x": "b" },
+ { "id": 5, "x": "b" },
+ { "id": 6, "x": "b" }
+]);
+
+insert into t2 ([
+ { "id": 11, "y": "c" },
+ { "id": 12, "y": "c" }
+]);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/gby-cross-join/gby-cross-join.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/gby-cross-join/gby-cross-join.3.query.sqlpp
new file mode 100644
index 0000000..34ec706
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/gby-cross-join/gby-cross-join.3.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select t1.x, count(*) as cnt
+from t2, t1
+group by t1.x
+order by t1.x;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/gby-cross-join/gby-cross-join.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/gby-cross-join/gby-cross-join.3.adm
new file mode 100644
index 0000000..e1e3cf1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/gby-cross-join/gby-cross-join.3.adm
@@ -0,0 +1,2 @@
+{ "cnt": 6, "x": "a" }
+{ "cnt": 6, "x": "b" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 4c034ad..8d06f71 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -5323,6 +5323,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="group-by">
+ <compilation-unit name="gby-cross-join">
+ <output-dir compare="Text">gby-cross-join</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
<compilation-unit name="gby-nested-01">
<output-dir compare="Text">gby-nested-01</output-dir>
</compilation-unit>
[asterixdb] 14/23: [NO ISSUE][STO] Close datasets of flushed
indexes after recovery
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit acff469e4363dde3c0aafe3c388f62aad4aa87a7
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Wed May 12 16:41:02 2021 +0300
[NO ISSUE][STO] Close datasets of flushed indexes after recovery
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- After performing redo of a flush log on any index, close its dataset
to ensure any cached state that might have been changed during recovery
is cleared (e.g. the component id generator).
- Fix LSMFlushRecoveryTest total number of records to be inserted.
- Update LSMFlushRecoveryTest to check for duplicate component ids.
Change-Id: I29072f475cc7b4d7d6efde415be0329fc568443e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11423
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Michael Blow <mb...@apache.org>
(cherry picked from commit 0e7e4bdb2c514fe7469a775c2ffc3d9e78e6a317)
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12024
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../org/apache/asterix/app/nc/RecoveryManager.java | 20 +++++++++++++-------
.../asterix/test/dataflow/LSMFlushRecoveryTest.java | 12 ++++++++++--
.../asterix/common/api/IDatasetLifecycleManager.java | 9 +++++++++
.../common/context/DatasetLifecycleManager.java | 11 +++++++++++
.../common/transactions/IRecoveryManager.java | 11 -----------
5 files changed, 43 insertions(+), 20 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index 1461ef4..718e4b3 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -173,15 +173,14 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
deleteRecoveryTemporaryFiles();
//get active partitions on this node
- replayPartitionsLogs(partitions, logMgr.getLogReader(true), lowWaterMarkLSN);
+ replayPartitionsLogs(partitions, logMgr.getLogReader(true), lowWaterMarkLSN, true);
}
- @Override
- public synchronized void replayPartitionsLogs(Set<Integer> partitions, ILogReader logReader, long lowWaterMarkLSN)
- throws IOException, ACIDException {
+ public synchronized void replayPartitionsLogs(Set<Integer> partitions, ILogReader logReader, long lowWaterMarkLSN,
+ boolean closeOnFlushRedo) throws IOException, ACIDException {
try {
Set<Long> winnerJobSet = startRecoverysAnalysisPhase(partitions, logReader, lowWaterMarkLSN);
- startRecoveryRedoPhase(partitions, logReader, lowWaterMarkLSN, winnerJobSet);
+ startRecoveryRedoPhase(partitions, logReader, lowWaterMarkLSN, winnerJobSet, closeOnFlushRedo);
} finally {
logReader.close();
deleteRecoveryTemporaryFiles();
@@ -277,7 +276,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
}
private synchronized void startRecoveryRedoPhase(Set<Integer> partitions, ILogReader logReader,
- long lowWaterMarkLSN, Set<Long> winnerTxnSet) throws IOException, ACIDException {
+ long lowWaterMarkLSN, Set<Long> winnerTxnSet, boolean closeOnFlushRedo) throws IOException, ACIDException {
int redoCount = 0;
long txnId = 0;
@@ -299,6 +298,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
TxnEntityId tempKeyTxnEntityId = new TxnEntityId(-1, -1, -1, null, -1, false);
ILogRecord logRecord = null;
+ Set<Integer> flushRedoDatasets = new HashSet<>();
try {
logReader.setPosition(lowWaterMarkLSN);
logRecord = logReader.next();
@@ -409,6 +409,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
&& !index.isCurrentMutableComponentEmpty()) {
// schedule flush
redoFlush(index, logRecord);
+ flushRedoDatasets.add(datasetId);
redoCount++;
} else {
// TODO: update checkpoint file?
@@ -441,6 +442,11 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
for (long r : resourceIdList) {
datasetLifecycleManager.close(resourcesMap.get(r).getPath());
}
+ if (closeOnFlushRedo) {
+ // close datasets of indexes to ensure any cached state that might've been changed by recovery is cleared
+ // e.g. when redoing a flush, the component id generator needs to be reinitialized
+ datasetLifecycleManager.closeDatasets(flushRedoDatasets);
+ }
}
}
@@ -525,7 +531,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
if (minLSN < readableSmallestLSN) {
minLSN = readableSmallestLSN;
}
- replayPartitionsLogs(partitions, logMgr.getLogReader(true), minLSN);
+ replayPartitionsLogs(partitions, logMgr.getLogReader(true), minLSN, false);
if (flush) {
appCtx.getDatasetLifecycleManager().flushAllDatasets();
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
index 03ca1f0..dd0bd9a 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
@@ -23,7 +23,9 @@ import java.lang.reflect.Field;
import java.rmi.RemoteException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.Semaphore;
import org.apache.asterix.app.bootstrap.TestNodeController;
@@ -62,6 +64,7 @@ import org.apache.hyracks.storage.am.lsm.btree.impl.AllowTestOpCallback;
import org.apache.hyracks.storage.am.lsm.btree.impl.ITestOpCallback;
import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
import org.apache.hyracks.storage.am.lsm.common.api.IIoOperationFailedCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentId;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -146,8 +149,7 @@ public class LSMFlushRecoveryTest {
checkComponentIds();
// insert more records
createInsertOps();
- insertRecords(PARTITION_0, StorageTestUtils.RECORDS_PER_COMPONENT, StorageTestUtils.RECORDS_PER_COMPONENT,
- true);
+ insertRecords(PARTITION_0, StorageTestUtils.TOTAL_NUM_OF_RECORDS, StorageTestUtils.RECORDS_PER_COMPONENT, true);
dsInfo.waitForIO();
checkComponentIds();
@@ -486,8 +488,14 @@ public class LSMFlushRecoveryTest {
List<ILSMDiskComponent> secondaryDiskComponents = secondaryIndexes[partitionIndex].getDiskComponents();
Assert.assertEquals(primaryDiskComponents.size(), secondaryDiskComponents.size());
+ Set<ILSMComponentId> uniqueIds = new HashSet<>();
for (int i = 0; i < primaryDiskComponents.size(); i++) {
Assert.assertEquals(primaryDiskComponents.get(i).getId(), secondaryDiskComponents.get(i).getId());
+ ILSMComponentId id = primaryDiskComponents.get(i).getId();
+ boolean added = uniqueIds.add(id);
+ if (!added) {
+ throw new IllegalStateException("found duplicate component ids: " + id);
+ }
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
index 10eddb1..0aad368 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
@@ -19,6 +19,7 @@
package org.apache.asterix.common.api;
import java.util.List;
+import java.util.Set;
import java.util.function.Predicate;
import org.apache.asterix.common.context.DatasetInfo;
@@ -112,6 +113,14 @@ public interface IDatasetLifecycleManager extends IResourceLifecycleManager<IInd
List<IVirtualBufferCache> getVirtualBufferCaches(int datasetId, int ioDeviceNum);
/**
+ * Attempts to close the datasets in {@code datasetsToClose}
+ *
+ * @param datasetsToClose
+ * @throws HyracksDataException
+ */
+ void closeDatasets(Set<Integer> datasetsToClose) throws HyracksDataException;
+
+ /**
* Flushes then closes all open datasets
*/
void closeAllDatasets() throws HyracksDataException;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
index d396d9b..fdf7822 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
@@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Predicate;
@@ -474,6 +475,16 @@ public class DatasetLifecycleManager implements IDatasetLifecycleManager, ILifeC
}
@Override
+ public synchronized void closeDatasets(Set<Integer> datasetsToClose) throws HyracksDataException {
+ ArrayList<DatasetResource> openDatasets = new ArrayList<>(datasets.values());
+ for (DatasetResource dsr : openDatasets) {
+ if (dsr.isOpen() && datasetsToClose.contains(dsr.getDatasetID())) {
+ closeDataset(dsr);
+ }
+ }
+ }
+
+ @Override
public synchronized void closeAllDatasets() throws HyracksDataException {
ArrayList<DatasetResource> openDatasets = new ArrayList<>(datasets.values());
for (DatasetResource dsr : openDatasets) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java
index bfe7963..8a5f34e 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java
@@ -85,17 +85,6 @@ public interface IRecoveryManager {
long getLocalMinFirstLSN() throws HyracksDataException;
/**
- * Replay the logs that belong to the passed {@code partitions} starting from the {@code lowWaterMarkLSN}
- *
- * @param partitions
- * @param lowWaterMarkLSN
- * @throws IOException
- * @throws ACIDException
- */
- void replayPartitionsLogs(Set<Integer> partitions, ILogReader logReader, long lowWaterMarkLSN)
- throws IOException, ACIDException;
-
- /**
* Creates a temporary file to be used during recovery
*
* @param txnId
[asterixdb] 23/23: Merge "Merge branch 'gerrit/cheshire-cat' into
'master'"
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit bb9b8f728ef5414f66da35e22f841fb29c176cae
Merge: a5c6e93 9d6c228
Author: Apache AsterixDB Gerrit <de...@asterixdb.apache.org>
AuthorDate: Wed Jun 30 00:06:20 2021 +0000
Merge "Merge branch 'gerrit/cheshire-cat' into 'master'"
.../optimizer/rules/PushGroupByThroughProduct.java | 17 +++---
.../api/http/server/NCQueryServiceServlet.java | 5 +-
.../api/http/server/QueryServiceServlet.java | 15 ++---
.../message/ExecuteStatementRequestMessage.java | 5 +-
.../asterix/app/translator/QueryTranslator.java | 3 +-
.../gby-cross-join/gby-cross-join.1.ddl.sqlpp | 34 +++++++++++
.../gby-cross-join/gby-cross-join.2.update.sqlpp | 34 +++++++++++
.../gby-cross-join/gby-cross-join.3.query.sqlpp | 25 ++++++++
.../window/misc_01/misc_01.10.query.sqlpp | 34 +++++++++++
.../group-by/gby-cross-join/gby-cross-join.3.adm | 2 +
.../results/window/misc_01/misc_01.10.adm | 4 ++
.../window/misc_01/misc_01.10.ast | 71 ++++++++++++++++++++++
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 ++
.../SqlppWindowAggregationSugarVisitor.java | 7 ++-
asterixdb/pom.xml | 4 +-
15 files changed, 241 insertions(+), 24 deletions(-)
[asterixdb] 12/23: [ASTERIXDB-2843][COMP] Fix type computer for
scalar aggregates
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit bcde0906d8a9771dd0abbb343e1b76c35c4bc95d
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Thu Jun 17 10:34:16 2021 -0700
[ASTERIXDB-2843][COMP] Fix type computer for scalar aggregates
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Align type computation for scalar aggregates with
regular aggregates
Change-Id: Ic587f29840262f052f15e7d257c3d7dda7e40686
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11984
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Reviewed-by: Till Westmann <ti...@apache.org>
---
.../asterix/om/functions/BuiltinFunctions.java | 29 +++++++------
...puter.java => AggregateResultTypeComputer.java} | 28 +++++-------
.../typecomputer/impl/MinMaxAggTypeComputer.java | 3 +-
.../impl/NumericSumAggTypeComputer.java | 33 ++------------
.../impl/ScalarVersionOfAggregateResultType.java | 50 ++++++++++++++--------
5 files changed, 65 insertions(+), 78 deletions(-)
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
index bf2709a..f2a1edf 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
@@ -1771,6 +1771,11 @@ public class BuiltinFunctions {
addFunction(NEGINF_IF, DoubleIfTypeComputer.INSTANCE, true);
// Aggregate Functions
+ ScalarVersionOfAggregateResultType scalarNumericSumTypeComputer =
+ new ScalarVersionOfAggregateResultType(NumericSumAggTypeComputer.INSTANCE);
+ ScalarVersionOfAggregateResultType scalarMinMaxTypeComputer =
+ new ScalarVersionOfAggregateResultType(MinMaxAggTypeComputer.INSTANCE);
+
addPrivateFunction(LISTIFY, OrderedListConstructorTypeComputer.INSTANCE, true);
addFunction(SCALAR_ARRAYAGG, ScalarArrayAggTypeComputer.INSTANCE, true);
addFunction(MAX, MinMaxAggTypeComputer.INSTANCE, true);
@@ -1816,7 +1821,7 @@ public class BuiltinFunctions {
// SUM
addFunction(SUM, NumericSumAggTypeComputer.INSTANCE, true);
- addFunction(SCALAR_SUM, ScalarVersionOfAggregateResultType.INSTANCE, true);
+ addFunction(SCALAR_SUM, scalarNumericSumTypeComputer, true);
addPrivateFunction(LOCAL_SUM, NumericSumAggTypeComputer.INSTANCE, true);
addPrivateFunction(INTERMEDIATE_SUM, NumericSumAggTypeComputer.INSTANCE, true);
addPrivateFunction(GLOBAL_SUM, NumericSumAggTypeComputer.INSTANCE, true);
@@ -1832,8 +1837,8 @@ public class BuiltinFunctions {
addPrivateFunction(SERIAL_INTERMEDIATE_SQL_AVG, LocalAvgTypeComputer.INSTANCE, true);
addFunction(SCALAR_AVG, NullableDoubleTypeComputer.INSTANCE, true);
addFunction(SCALAR_COUNT, AInt64TypeComputer.INSTANCE, true);
- addFunction(SCALAR_MAX, ScalarVersionOfAggregateResultType.INSTANCE, true);
- addFunction(SCALAR_MIN, ScalarVersionOfAggregateResultType.INSTANCE, true);
+ addFunction(SCALAR_MAX, scalarMinMaxTypeComputer, true);
+ addFunction(SCALAR_MIN, scalarMinMaxTypeComputer, true);
addPrivateFunction(INTERMEDIATE_AVG, LocalAvgTypeComputer.INSTANCE, true);
addFunction(SCALAR_STDDEV_SAMP, NullableDoubleTypeComputer.INSTANCE, true);
addPrivateFunction(INTERMEDIATE_STDDEV_SAMP, LocalSingleVarStatisticsTypeComputer.INSTANCE, true);
@@ -1874,7 +1879,7 @@ public class BuiltinFunctions {
// SQL SUM
addFunction(SQL_SUM, NumericSumAggTypeComputer.INSTANCE, true);
- addFunction(SCALAR_SQL_SUM, ScalarVersionOfAggregateResultType.INSTANCE, true);
+ addFunction(SCALAR_SQL_SUM, scalarNumericSumTypeComputer, true);
addPrivateFunction(LOCAL_SQL_SUM, NumericSumAggTypeComputer.INSTANCE, true);
addPrivateFunction(INTERMEDIATE_SQL_SUM, NumericSumAggTypeComputer.INSTANCE, true);
addPrivateFunction(GLOBAL_SQL_SUM, NumericSumAggTypeComputer.INSTANCE, true);
@@ -1898,8 +1903,8 @@ public class BuiltinFunctions {
addPrivateFunction(GLOBAL_SQL_MIN, MinMaxAggTypeComputer.INSTANCE, true);
addFunction(SCALAR_SQL_AVG, NullableDoubleTypeComputer.INSTANCE, true);
addFunction(SCALAR_SQL_COUNT, AInt64TypeComputer.INSTANCE, true);
- addFunction(SCALAR_SQL_MAX, ScalarVersionOfAggregateResultType.INSTANCE, true);
- addFunction(SCALAR_SQL_MIN, ScalarVersionOfAggregateResultType.INSTANCE, true);
+ addFunction(SCALAR_SQL_MAX, scalarMinMaxTypeComputer, true);
+ addFunction(SCALAR_SQL_MIN, scalarMinMaxTypeComputer, true);
addPrivateFunction(INTERMEDIATE_SQL_AVG, LocalAvgTypeComputer.INSTANCE, true);
addFunction(SQL_STDDEV_SAMP, NullableDoubleTypeComputer.INSTANCE, true);
addPrivateFunction(GLOBAL_SQL_STDDEV_SAMP, NullableDoubleTypeComputer.INSTANCE, true);
@@ -1974,9 +1979,9 @@ public class BuiltinFunctions {
addFunction(SCALAR_SQL_COUNT_DISTINCT, AInt64TypeComputer.INSTANCE, true);
addFunction(SUM_DISTINCT, NumericSumAggTypeComputer.INSTANCE, true);
- addFunction(SCALAR_SUM_DISTINCT, ScalarVersionOfAggregateResultType.INSTANCE, true);
+ addFunction(SCALAR_SUM_DISTINCT, scalarNumericSumTypeComputer, true);
addFunction(SQL_SUM_DISTINCT, NumericSumAggTypeComputer.INSTANCE, true);
- addFunction(SCALAR_SQL_SUM_DISTINCT, ScalarVersionOfAggregateResultType.INSTANCE, true);
+ addFunction(SCALAR_SQL_SUM_DISTINCT, scalarNumericSumTypeComputer, true);
addFunction(AVG_DISTINCT, NullableDoubleTypeComputer.INSTANCE, true);
addFunction(SCALAR_AVG_DISTINCT, NullableDoubleTypeComputer.INSTANCE, true);
@@ -1984,14 +1989,14 @@ public class BuiltinFunctions {
addFunction(SCALAR_SQL_AVG_DISTINCT, NullableDoubleTypeComputer.INSTANCE, true);
addFunction(MAX_DISTINCT, MinMaxAggTypeComputer.INSTANCE, true);
- addFunction(SCALAR_MAX_DISTINCT, ScalarVersionOfAggregateResultType.INSTANCE, true);
+ addFunction(SCALAR_MAX_DISTINCT, scalarMinMaxTypeComputer, true);
addFunction(SQL_MAX_DISTINCT, MinMaxAggTypeComputer.INSTANCE, true);
- addFunction(SCALAR_SQL_MAX_DISTINCT, ScalarVersionOfAggregateResultType.INSTANCE, true);
+ addFunction(SCALAR_SQL_MAX_DISTINCT, scalarMinMaxTypeComputer, true);
addFunction(MIN_DISTINCT, MinMaxAggTypeComputer.INSTANCE, true);
- addFunction(SCALAR_MIN_DISTINCT, ScalarVersionOfAggregateResultType.INSTANCE, true);
+ addFunction(SCALAR_MIN_DISTINCT, scalarMinMaxTypeComputer, true);
addFunction(SQL_MIN_DISTINCT, MinMaxAggTypeComputer.INSTANCE, true);
- addFunction(SCALAR_SQL_MIN_DISTINCT, ScalarVersionOfAggregateResultType.INSTANCE, true);
+ addFunction(SCALAR_SQL_MIN_DISTINCT, scalarMinMaxTypeComputer, true);
addFunction(STDDEV_SAMP_DISTINCT, NullableDoubleTypeComputer.INSTANCE, true);
addFunction(SCALAR_STDDEV_SAMP_DISTINCT, NullableDoubleTypeComputer.INSTANCE, true);
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/MinMaxAggTypeComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/AggregateResultTypeComputer.java
similarity index 57%
copy from asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/MinMaxAggTypeComputer.java
copy to asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/AggregateResultTypeComputer.java
index c34b5ed..8e663a7 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/MinMaxAggTypeComputer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/AggregateResultTypeComputer.java
@@ -16,30 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.asterix.om.typecomputer.impl;
-import org.apache.asterix.dataflow.data.common.ILogicalBinaryComparator;
import org.apache.asterix.om.typecomputer.base.AbstractResultTypeComputer;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.AUnionType;
-import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.api.exceptions.SourceLocation;
-public class MinMaxAggTypeComputer extends AbstractResultTypeComputer {
-
- public static final MinMaxAggTypeComputer INSTANCE = new MinMaxAggTypeComputer();
-
- private MinMaxAggTypeComputer() {
+public abstract class AggregateResultTypeComputer extends AbstractResultTypeComputer {
+ @Override
+ protected void checkArgType(FunctionIdentifier funcId, int argIndex, IAType type, SourceLocation sourceLoc)
+ throws AlgebricksException {
+ super.checkArgType(funcId, argIndex, type, sourceLoc);
}
@Override
- protected IAType getResultType(ILogicalExpression expr, IAType... strippedInputTypes) throws AlgebricksException {
- ATypeTag typeTag = strippedInputTypes[0].getTypeTag();
- if (ILogicalBinaryComparator.inequalityUndefined(typeTag)) {
- return BuiltinType.ANULL;
- }
- return typeTag == ATypeTag.ANY ? BuiltinType.ANY : AUnionType.createUnknownableType(strippedInputTypes[0]);
- }
-}
+ protected abstract IAType getResultType(ILogicalExpression expr, IAType... strippedInputTypes)
+ throws AlgebricksException;
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/MinMaxAggTypeComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/MinMaxAggTypeComputer.java
index c34b5ed..fc1eee5 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/MinMaxAggTypeComputer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/MinMaxAggTypeComputer.java
@@ -19,7 +19,6 @@
package org.apache.asterix.om.typecomputer.impl;
import org.apache.asterix.dataflow.data.common.ILogicalBinaryComparator;
-import org.apache.asterix.om.typecomputer.base.AbstractResultTypeComputer;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.AUnionType;
import org.apache.asterix.om.types.BuiltinType;
@@ -27,7 +26,7 @@ import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-public class MinMaxAggTypeComputer extends AbstractResultTypeComputer {
+public class MinMaxAggTypeComputer extends AggregateResultTypeComputer {
public static final MinMaxAggTypeComputer INSTANCE = new MinMaxAggTypeComputer();
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/NumericSumAggTypeComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/NumericSumAggTypeComputer.java
index 1c67e56..a4b5e34 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/NumericSumAggTypeComputer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/NumericSumAggTypeComputer.java
@@ -18,42 +18,20 @@
*/
package org.apache.asterix.om.typecomputer.impl;
-import org.apache.asterix.om.exceptions.UnsupportedTypeException;
-import org.apache.asterix.om.typecomputer.base.AbstractResultTypeComputer;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.AUnionType;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.api.exceptions.SourceLocation;
-public class NumericSumAggTypeComputer extends AbstractResultTypeComputer {
+public class NumericSumAggTypeComputer extends AggregateResultTypeComputer {
public static final NumericSumAggTypeComputer INSTANCE = new NumericSumAggTypeComputer();
private NumericSumAggTypeComputer() {
}
@Override
- protected void checkArgType(FunctionIdentifier funcId, int argIndex, IAType type, SourceLocation sourceLoc)
- throws AlgebricksException {
- ATypeTag tag = type.getTypeTag();
- switch (tag) {
- case DOUBLE:
- case FLOAT:
- case BIGINT:
- case INTEGER:
- case SMALLINT:
- case TINYINT:
- case ANY:
- break;
- default:
- throw new UnsupportedTypeException(sourceLoc, funcId, tag);
- }
- }
-
- @Override
protected IAType getResultType(ILogicalExpression expr, IAType... strippedInputTypes) throws AlgebricksException {
ATypeTag tag = strippedInputTypes[0].getTypeTag();
switch (tag) {
@@ -61,15 +39,12 @@ public class NumericSumAggTypeComputer extends AbstractResultTypeComputer {
case SMALLINT:
case INTEGER:
case BIGINT:
- IAType int64Type = BuiltinType.AINT64;
- return AUnionType.createNullableType(int64Type, "AggResult");
+ return AUnionType.createNullableType(BuiltinType.AINT64);
case FLOAT:
case DOUBLE:
- IAType doubleType = BuiltinType.ADOUBLE;
- return AUnionType.createNullableType(doubleType, "AggResult");
+ return AUnionType.createNullableType(BuiltinType.ADOUBLE);
case ANY:
- IAType anyType = strippedInputTypes[0];
- return AUnionType.createNullableType(anyType, "AggResult");
+ return BuiltinType.ANY;
default:
// All other possible cases.
return BuiltinType.ANULL;
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/ScalarVersionOfAggregateResultType.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/ScalarVersionOfAggregateResultType.java
index 5b90974..fda0285 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/ScalarVersionOfAggregateResultType.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/ScalarVersionOfAggregateResultType.java
@@ -18,9 +18,7 @@
*/
package org.apache.asterix.om.typecomputer.impl;
-import org.apache.asterix.om.exceptions.TypeMismatchException;
import org.apache.asterix.om.typecomputer.base.AbstractResultTypeComputer;
-import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.AUnionType;
import org.apache.asterix.om.types.AbstractCollectionType;
import org.apache.asterix.om.types.BuiltinType;
@@ -32,32 +30,48 @@ import org.apache.hyracks.api.exceptions.SourceLocation;
public class ScalarVersionOfAggregateResultType extends AbstractResultTypeComputer {
- public static final ScalarVersionOfAggregateResultType INSTANCE = new ScalarVersionOfAggregateResultType();
+ private final AggregateResultTypeComputer aggResultTypeComputer;
- private ScalarVersionOfAggregateResultType() {
+ public ScalarVersionOfAggregateResultType(AggregateResultTypeComputer aggResultTypeComputer) {
+ this.aggResultTypeComputer = aggResultTypeComputer;
}
@Override
- public void checkArgType(FunctionIdentifier funcId, int argIndex, IAType type, SourceLocation sourceLoc)
+ protected void checkArgType(FunctionIdentifier funcId, int argIndex, IAType argType, SourceLocation sourceLoc)
throws AlgebricksException {
- ATypeTag tag = type.getTypeTag();
- if (tag != ATypeTag.ANY && tag != ATypeTag.ARRAY && tag != ATypeTag.MULTISET) {
- throw new TypeMismatchException(sourceLoc, funcId, argIndex, tag, ATypeTag.ARRAY, ATypeTag.MULTISET);
+ if (argIndex == 0) {
+ switch (argType.getTypeTag()) {
+ case ARRAY:
+ case MULTISET:
+ AbstractCollectionType act = (AbstractCollectionType) argType;
+ aggResultTypeComputer.checkArgType(funcId, argIndex, act.getItemType(), sourceLoc);
+ break;
+ }
}
}
@Override
protected IAType getResultType(ILogicalExpression expr, IAType... strippedInputTypes) throws AlgebricksException {
- ATypeTag tag = strippedInputTypes[0].getTypeTag();
- if (tag == ATypeTag.ANY) {
- return BuiltinType.ANY;
+ IAType argType = strippedInputTypes[0];
+ switch (argType.getTypeTag()) {
+ case ARRAY:
+ case MULTISET:
+ AbstractCollectionType act = (AbstractCollectionType) argType;
+ IAType[] strippedInputTypes2 = strippedInputTypes.clone();
+ strippedInputTypes2[0] = act.getItemType();
+ IAType resultType = aggResultTypeComputer.getResultType(expr, strippedInputTypes2);
+ switch (resultType.getTypeTag()) {
+ case NULL:
+ case MISSING:
+ case ANY:
+ return resultType;
+ case UNION:
+ return AUnionType.createUnknownableType(((AUnionType) resultType).getActualType());
+ default:
+ return AUnionType.createUnknownableType(resultType);
+ }
+ default:
+ return BuiltinType.ANY;
}
- if (tag != ATypeTag.ARRAY && tag != ATypeTag.MULTISET) {
- // this condition being true would've thrown an exception above, no?
- return strippedInputTypes[0];
- }
- AbstractCollectionType act = (AbstractCollectionType) strippedInputTypes[0];
- IAType t = act.getItemType();
- return AUnionType.createUnknownableType(t);
}
}
[asterixdb] 04/23: [ASTERIXDB-2804][FUN] Use pseudo-random for
random(seed)
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d28c0a7669f1d4a877524011517aea0fe2ff2505
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Thu Jun 3 20:43:22 2021 +0300
[ASTERIXDB-2804][FUN] Use pseudo-random for random(seed)
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Use pseudo-random for random(seed)
Change-Id: I14e81b24933744b136f85bd1218987401830fad9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11763
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
---
.../queries_sqlpp/misc/random/random.3.query.sqlpp | 22 +++++++++++
.../runtimets/results/misc/random/random.3.adm | 3 ++
.../functions/RandomWithSeedDescriptor.java | 5 ++-
.../evaluators/functions/utils/RandomHelper.java | 46 +++++++++++-----------
4 files changed, 51 insertions(+), 25 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/random/random.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/random/random.3.query.sqlpp
new file mode 100644
index 0000000..8b0aa23
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/random/random.3.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+FROM range(1, 3) AS r
+SELECT random(int8("12")) AS i8, random(int16("12")) AS i16, random(int32("12")) AS i32, random(int64("12")) AS i64,
+random(float("12")) AS float, random(double("12")) AS double;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/random/random.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/random/random.3.adm
new file mode 100644
index 0000000..5b9e73c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/random/random.3.adm
@@ -0,0 +1,3 @@
+{ "i8": 0.26795443606823943, "i16": 0.26795443606823943, "i32": 0.26795443606823943, "i64": 0.26795443606823943, "float": 0.26795443606823943, "double": 0.26795443606823943 }
+{ "i8": 0.4533526797678967, "i16": 0.4533526797678967, "i32": 0.4533526797678967, "i64": 0.4533526797678967, "float": 0.4533526797678967, "double": 0.4533526797678967 }
+{ "i8": 0.38508513586474447, "i16": 0.38508513586474447, "i32": 0.38508513586474447, "i64": 0.38508513586474447, "float": 0.38508513586474447, "double": 0.38508513586474447 }
\ No newline at end of file
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/RandomWithSeedDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/RandomWithSeedDescriptor.java
index aa49010..bd74f24 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/RandomWithSeedDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/RandomWithSeedDescriptor.java
@@ -24,6 +24,7 @@ import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.functions.IFunctionDescriptor;
import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
import org.apache.asterix.runtime.evaluators.functions.utils.RandomHelper;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
@@ -78,7 +79,9 @@ public class RandomWithSeedDescriptor extends AbstractScalarFunctionDynamicDescr
case BIGINT:
case FLOAT:
case DOUBLE:
- randomHelper.setSeed(bytes, offset + 1, arg0.getLength() - 1);
+ double seed =
+ ATypeHierarchy.getDoubleValue(getIdentifier().getName(), 0, bytes, offset);
+ randomHelper.setSeed(seed);
randomHelper.nextDouble(resultPointable);
break;
default:
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/utils/RandomHelper.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/utils/RandomHelper.java
index 7ed1ce5..bc905ba 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/utils/RandomHelper.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/utils/RandomHelper.java
@@ -20,7 +20,7 @@
package org.apache.asterix.runtime.evaluators.functions.utils;
import java.io.DataOutput;
-import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
@@ -30,44 +30,42 @@ import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IPointable;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.data.std.util.DataUtils;
-import org.apache.hyracks.data.std.util.GrowableArray;
public final class RandomHelper {
- private final SecureRandom random = new SecureRandom();
+ private final SecureRandom random;
- private final GrowableArray seed;
+ private double seed;
+
+ private boolean isFirst;
private final AMutableDouble aDouble = new AMutableDouble(0);
@SuppressWarnings("rawtypes")
- private ISerializerDeserializer doubleSerde =
+ private final ISerializerDeserializer doubleSerde =
SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ADOUBLE);
private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
private final DataOutput dataOutput = resultStorage.getDataOutput();
- public RandomHelper(boolean withSeed) {
- seed = withSeed ? new GrowableArray(8) : null;
- }
-
- public void setSeed(byte[] bytes, int offset, int length) throws HyracksDataException {
- if (seed == null) {
- throw new IllegalStateException();
- }
-
- boolean sameSeed =
- seed.getLength() == length && DataUtils.equalsInRange(seed.getByteArray(), 0, bytes, offset, length);
-
- if (!sameSeed) {
+ public RandomHelper(boolean withSeed) throws HyracksDataException {
+ if (withSeed) {
try {
- seed.reset();
- seed.append(bytes, offset, length);
- random.setSeed(seed.getByteArray());
- } catch (IOException e) {
- throw HyracksDataException.create(e);
+ random = SecureRandom.getInstance("SHA1PRNG");
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("random()");
}
+ } else {
+ random = new SecureRandom();
+ }
+ isFirst = true;
+ }
+
+ public void setSeed(double seedVal) throws HyracksDataException {
+ if (isFirst || seedVal != seed) {
+ seed = seedVal;
+ isFirst = false;
+ random.setSeed(Double.doubleToLongBits(seedVal));
}
}
[asterixdb] 07/23: [ASTERIXDB-2857][RT] Incorrect result for nested
loop outer join
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 642fa1049a2e2c372a1d8d54c1dc7e5d85cad6d8
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Tue Mar 30 17:12:27 2021 -0700
[ASTERIXDB-2857][RT] Incorrect result for nested loop outer join
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Fix incorrect number of unmatched tuples emitted
by nested loop implementation of left outer join
- Add RunFileWriter.eraseClosed() method
Change-Id: Id48c2f25acf85e61bb1112811ecfe544a0edae86
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11825
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../leftouterjoin/query-ASTERIXDB-2857.sqlpp | 63 ++++++++
.../leftouterjoin/query-ASTERIXDB-2857.plan | 39 +++++
.../query-ASTERIXDB-2857.1.ddl.sqlpp | 48 ++++++
.../query-ASTERIXDB-2857.2.update.sqlpp | 22 +++
.../query-ASTERIXDB-2857.3.query.sqlpp | 39 +++++
.../query-ASTERIXDB-2857.3.adm | 2 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 +
.../hyracks/dataflow/common/io/RunFileWriter.java | 19 ++-
.../hyracks/dataflow/std/join/NestedLoopJoin.java | 165 ++++++++++++---------
.../OptimizedHybridHashJoinOperatorDescriptor.java | 5 +-
10 files changed, 332 insertions(+), 75 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/leftouterjoin/query-ASTERIXDB-2857.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/leftouterjoin/query-ASTERIXDB-2857.sqlpp
new file mode 100644
index 0000000..2304349
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/leftouterjoin/query-ASTERIXDB-2857.sqlpp
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test nested loop implementation of left outer join
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type tenkType as closed {
+ unique1 : integer,
+ unique2 : integer,
+ two : integer,
+ four : integer,
+ ten : integer,
+ twenty : integer,
+ hundred : integer,
+ thousand : integer,
+ twothousand : integer,
+ fivethous : integer,
+ tenthous : integer,
+ odd100 : integer,
+ even100 : integer,
+ stringu1 : string,
+ stringu2 : string,
+ string4 : string
+};
+
+create dataset tenk(tenkType) primary key unique2;
+
+SELECT
+ t0.unique1 AS t0_unique1,
+ t1.unique1 AS t1_unique1,
+ t2.unique1 AS t2_unique1
+FROM (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 2
+) t0
+INNER JOIN (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 4
+) t1 ON t0.unique2 = t1.unique2
+LEFT JOIN (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 6
+) t2 ON t0.unique2 + t2.unique2 = 2 * t1.unique2
+ORDER BY t0_unique1, t1_unique1, t2_unique1;
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/query-ASTERIXDB-2857.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/query-ASTERIXDB-2857.plan
new file mode 100644
index 0000000..0df2a49
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/leftouterjoin/query-ASTERIXDB-2857.plan
@@ -0,0 +1,39 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$127(ASC), $$128(ASC), $#3(ASC) ] |PARTITIONED|
+ -- STABLE_SORT [$$127(ASC), $$128(ASC), $#3(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- NESTED_LOOP |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$118][$$119] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.1.ddl.sqlpp
new file mode 100644
index 0000000..87b5d75
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.1.ddl.sqlpp
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test nested loop implementation of left outer join
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type tenkType as closed {
+ unique1 : integer,
+ unique2 : integer,
+ two : integer,
+ four : integer,
+ ten : integer,
+ twenty : integer,
+ hundred : integer,
+ thousand : integer,
+ twothousand : integer,
+ fivethous : integer,
+ tenthous : integer,
+ odd100 : integer,
+ even100 : integer,
+ stringu1 : string,
+ stringu2 : string,
+ string4 : string
+};
+
+create dataset tenk(tenkType) primary key unique2;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.2.update.sqlpp
new file mode 100644
index 0000000..2d7e768
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.2.update.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+load dataset tenk using localfs ((`path`=`asterix_nc1://data/tenk.tbl`),(`format`=`delimited-text`),(`delimiter`=`|`));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.query.sqlpp
new file mode 100644
index 0000000..823a540
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.query.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test nested loop implementation of left outer join
+ */
+
+USE test;
+
+SELECT
+ t0.unique1 AS t0_unique1,
+ t1.unique1 AS t1_unique1,
+ t2.unique1 AS t2_unique1
+FROM (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 2
+) t0
+INNER JOIN (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 4
+) t1 ON t0.unique2 = t1.unique2
+LEFT JOIN (
+ SELECT unique1, unique2 FROM tenk WHERE unique2 < 6
+) t2 ON t0.unique2 + t2.unique2 = 2 * t1.unique2
+ORDER BY t0_unique1, t1_unique1, t2_unique1;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.adm
new file mode 100644
index 0000000..1a31db8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/leftouterjoin/query-ASTERIXDB-2857/query-ASTERIXDB-2857.3.adm
@@ -0,0 +1,2 @@
+{ "t0_unique1": 1891, "t1_unique1": 1891, "t2_unique1": 1891 }
+{ "t0_unique1": 8800, "t1_unique1": 8800, "t2_unique1": 8800 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index af6ca78..0015fa6 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -11897,6 +11897,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="leftouterjoin">
+ <compilation-unit name="query-ASTERIXDB-2857">
+ <output-dir compare="Text">query-ASTERIXDB-2857</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="leftouterjoin">
<compilation-unit name="right_branch_opt_1">
<output-dir compare="Text">right_branch_opt_1</output-dir>
</compilation-unit>
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/RunFileWriter.java b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/RunFileWriter.java
index dabdd4f..0dbd127 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/RunFileWriter.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/RunFileWriter.java
@@ -75,12 +75,21 @@ public class RunFileWriter implements IFrameWriter {
}
public void erase() throws HyracksDataException {
- close();
- file.delete();
+ try {
+ close();
+ } finally {
+ // Make sure we never access the file if it is deleted.
+ handle = null;
+ eraseClosed();
+ }
+ }
- // Make sure we never access the file if it is deleted.
- file = null;
- handle = null;
+ public void eraseClosed() {
+ FileReference f = file;
+ if (f != null) {
+ file = null;
+ f.delete();
+ }
}
public FileReference getFileReference() {
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/NestedLoopJoin.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/NestedLoopJoin.java
index 2eae25c..dbc91c2 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/NestedLoopJoin.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/NestedLoopJoin.java
@@ -20,6 +20,7 @@ package org.apache.hyracks.dataflow.std.join;
import java.io.DataOutput;
import java.nio.ByteBuffer;
+import java.util.BitSet;
import org.apache.hyracks.api.comm.IFrame;
import org.apache.hyracks.api.comm.IFrameWriter;
@@ -28,6 +29,7 @@ import org.apache.hyracks.api.context.IHyracksJobletContext;
import org.apache.hyracks.api.dataflow.value.IMissingWriter;
import org.apache.hyracks.api.dataflow.value.IPredicateEvaluator;
import org.apache.hyracks.api.dataflow.value.ITuplePairComparator;
+import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -43,6 +45,14 @@ import org.apache.hyracks.dataflow.std.buffermanager.VariableFrameMemoryManager;
import org.apache.hyracks.dataflow.std.buffermanager.VariableFramePool;
public class NestedLoopJoin {
+ // Note: Min memory budget should be less than {@code AbstractJoinPOperator.MIN_FRAME_LIMIT_FOR_JOIN}
+ // Inner join: 1 frame for the outer input side, 1 frame for the inner input side, 1 frame for the output
+ private static final int MIN_FRAME_BUDGET_INNER_JOIN = 3;
+ // Outer join extra: Add 1 frame for the {@code outerMatchLOJ} bitset
+ private static final int MIN_FRAME_BUDGET_OUTER_JOIN = MIN_FRAME_BUDGET_INNER_JOIN + 1;
+ // Outer join needs 1 bit per each tuple in the outer side buffer
+ private static final int ESTIMATE_AVG_TUPLE_SIZE = 128;
+
private final FrameTupleAccessor accessorInner;
private final FrameTupleAccessor accessorOuter;
private final FrameTupleAppender appender;
@@ -54,30 +64,45 @@ public class NestedLoopJoin {
private final boolean isLeftOuter;
private final ArrayTupleBuilder missingTupleBuilder;
private final IPredicateEvaluator predEvaluator;
- private boolean isReversed; //Added for handling correct calling for predicate-evaluator upon recursive calls (in OptimizedHybridHashJoin) that cause role-reversal
+ // Added for handling correct calling for predicate-evaluator upon recursive calls
+ // (in OptimizedHybridHashJoin) that cause role-reversal
+ private final boolean isReversed;
private final BufferInfo tempInfo = new BufferInfo(null, -1, -1);
+ private final BitSet outerMatchLOJ;
public NestedLoopJoin(IHyracksJobletContext jobletContext, FrameTupleAccessor accessorOuter,
- FrameTupleAccessor accessorInner, int memSize, IPredicateEvaluator predEval, boolean isLeftOuter,
+ FrameTupleAccessor accessorInner, int memBudgetInFrames, IPredicateEvaluator predEval, boolean isLeftOuter,
IMissingWriter[] missingWriters) throws HyracksDataException {
+ this(jobletContext, accessorOuter, accessorInner, memBudgetInFrames, predEval, isLeftOuter, missingWriters,
+ false);
+ }
+
+ public NestedLoopJoin(IHyracksJobletContext jobletContext, FrameTupleAccessor accessorOuter,
+ FrameTupleAccessor accessorInner, int memBudgetInFrames, IPredicateEvaluator predEval, boolean isLeftOuter,
+ IMissingWriter[] missingWriters, boolean isReversed) throws HyracksDataException {
this.accessorInner = accessorInner;
this.accessorOuter = accessorOuter;
this.appender = new FrameTupleAppender();
this.outBuffer = new VSizeFrame(jobletContext);
this.innerBuffer = new VSizeFrame(jobletContext);
this.appender.reset(outBuffer, true);
- if (memSize < 3) {
- throw new HyracksDataException("Not enough memory is available for Nested Loop Join");
+
+ int minMemBudgetInFrames = isLeftOuter ? MIN_FRAME_BUDGET_OUTER_JOIN : MIN_FRAME_BUDGET_INNER_JOIN;
+ if (memBudgetInFrames < minMemBudgetInFrames) {
+ throw new HyracksDataException(ErrorCode.INSUFFICIENT_MEMORY);
}
+ int outerBufferMngrMemBudgetInFrames = memBudgetInFrames - minMemBudgetInFrames + 1;
+ int outerBufferMngrMemBudgetInBytes = jobletContext.getInitialFrameSize() * outerBufferMngrMemBudgetInFrames;
this.outerBufferMngr = new VariableFrameMemoryManager(
- new VariableFramePool(jobletContext, jobletContext.getInitialFrameSize() * (memSize - 2)),
- FrameFreeSlotPolicyFactory.createFreeSlotPolicy(EnumFreeSlotPolicy.LAST_FIT, memSize - 2));
+ new VariableFramePool(jobletContext, outerBufferMngrMemBudgetInBytes), FrameFreeSlotPolicyFactory
+ .createFreeSlotPolicy(EnumFreeSlotPolicy.LAST_FIT, outerBufferMngrMemBudgetInFrames));
this.predEvaluator = predEval;
- this.isReversed = false;
-
this.isLeftOuter = isLeftOuter;
if (isLeftOuter) {
+ if (isReversed) {
+ throw new HyracksDataException(ErrorCode.INVALID_OPERATOR_OPERATION, "reverse", "outer join");
+ }
int innerFieldCount = this.accessorInner.getFieldCount();
missingTupleBuilder = new ArrayTupleBuilder(innerFieldCount);
DataOutput out = missingTupleBuilder.getDataOutput();
@@ -85,9 +110,14 @@ public class NestedLoopJoin {
missingWriters[i].writeMissing(out);
missingTupleBuilder.addFieldEndOffset();
}
+ // Outer join needs 1 bit per each tuple in the outer side buffer
+ int outerMatchLOJCardinalityEstimate = outerBufferMngrMemBudgetInBytes / ESTIMATE_AVG_TUPLE_SIZE;
+ outerMatchLOJ = new BitSet(Math.max(outerMatchLOJCardinalityEstimate, 1));
} else {
missingTupleBuilder = null;
+ outerMatchLOJ = null;
}
+ this.isReversed = isReversed;
FileReference file =
jobletContext.createManagedWorkspaceFile(this.getClass().getSimpleName() + this.toString());
@@ -110,23 +140,7 @@ public class NestedLoopJoin {
public void join(ByteBuffer outerBuffer, IFrameWriter writer) throws HyracksDataException {
if (outerBufferMngr.insertFrame(outerBuffer) < 0) {
- RunFileReader runFileReader = runFileWriter.createReader();
- try {
- runFileReader.open();
- if (runFileReader.nextFrame(innerBuffer)) {
- do {
- for (int i = 0; i < outerBufferMngr.getNumFrames(); i++) {
- blockJoin(outerBufferMngr.getFrame(i, tempInfo), innerBuffer.getBuffer(), writer);
- }
- } while (runFileReader.nextFrame(innerBuffer));
- } else if (isLeftOuter) {
- for (int i = 0; i < outerBufferMngr.getNumFrames(); i++) {
- appendMissing(outerBufferMngr.getFrame(i, tempInfo), writer);
- }
- }
- } finally {
- runFileReader.close();
- }
+ multiBlockJoin(writer);
outerBufferMngr.reset();
if (outerBufferMngr.insertFrame(outerBuffer) < 0) {
throw new HyracksDataException("The given outer frame of size:" + outerBuffer.capacity()
@@ -135,16 +149,51 @@ public class NestedLoopJoin {
}
}
- private void blockJoin(BufferInfo outerBufferInfo, ByteBuffer innerBuffer, IFrameWriter writer)
- throws HyracksDataException {
- accessorOuter.reset(outerBufferInfo.getBuffer(), outerBufferInfo.getStartOffset(), outerBufferInfo.getLength());
- accessorInner.reset(innerBuffer);
- int tupleCount0 = accessorOuter.getTupleCount();
- int tupleCount1 = accessorInner.getTupleCount();
+ private void multiBlockJoin(IFrameWriter writer) throws HyracksDataException {
+ int outerBufferFrameCount = outerBufferMngr.getNumFrames();
+ if (outerBufferFrameCount == 0) {
+ return;
+ }
+ RunFileReader runFileReader = runFileWriter.createReader();
+ try {
+ runFileReader.open();
+ if (isLeftOuter) {
+ outerMatchLOJ.clear();
+ }
+ while (runFileReader.nextFrame(innerBuffer)) {
+ int outerTupleRunningCount = 0;
+ for (int i = 0; i < outerBufferFrameCount; i++) {
+ BufferInfo outerBufferInfo = outerBufferMngr.getFrame(i, tempInfo);
+ accessorOuter.reset(outerBufferInfo.getBuffer(), outerBufferInfo.getStartOffset(),
+ outerBufferInfo.getLength());
+ int outerTupleCount = accessorOuter.getTupleCount();
+ accessorInner.reset(innerBuffer.getBuffer());
+ blockJoin(outerTupleRunningCount, writer);
+ outerTupleRunningCount += outerTupleCount;
+ }
+ }
+ if (isLeftOuter) {
+ int outerTupleRunningCount = 0;
+ for (int i = 0; i < outerBufferFrameCount; i++) {
+ BufferInfo outerBufferInfo = outerBufferMngr.getFrame(i, tempInfo);
+ accessorOuter.reset(outerBufferInfo.getBuffer(), outerBufferInfo.getStartOffset(),
+ outerBufferInfo.getLength());
+ int outerFrameTupleCount = accessorOuter.getTupleCount();
+ appendMissing(outerTupleRunningCount, outerFrameTupleCount, writer);
+ outerTupleRunningCount += outerFrameTupleCount;
+ }
+ }
+ } finally {
+ runFileReader.close();
+ }
+ }
- for (int i = 0; i < tupleCount0; ++i) {
+ private void blockJoin(int outerTupleStartPos, IFrameWriter writer) throws HyracksDataException {
+ int outerTupleCount = accessorOuter.getTupleCount();
+ int innerTupleCount = accessorInner.getTupleCount();
+ for (int i = 0; i < outerTupleCount; ++i) {
boolean matchFound = false;
- for (int j = 0; j < tupleCount1; ++j) {
+ for (int j = 0; j < innerTupleCount; ++j) {
int c = tpComparator.compare(accessorOuter, i, accessorInner, j);
boolean prdEval = evaluatePredicate(i, j);
if (c == 0 && prdEval) {
@@ -152,13 +201,8 @@ public class NestedLoopJoin {
appendToResults(i, j, writer);
}
}
-
- if (!matchFound && isLeftOuter) {
- final int[] ntFieldEndOffsets = missingTupleBuilder.getFieldEndOffsets();
- final byte[] ntByteArray = missingTupleBuilder.getByteArray();
- final int ntSize = missingTupleBuilder.getSize();
- FrameUtils.appendConcatToWriter(writer, appender, accessorOuter, i, ntFieldEndOffsets, ntByteArray, 0,
- ntSize);
+ if (isLeftOuter && matchFound) {
+ outerMatchLOJ.set(outerTupleStartPos + i);
}
}
}
@@ -184,15 +228,18 @@ public class NestedLoopJoin {
FrameUtils.appendConcatToWriter(writer, appender, accessor1, tupleId1, accessor2, tupleId2);
}
- private void appendMissing(BufferInfo outerBufferInfo, IFrameWriter writer) throws HyracksDataException {
- accessorOuter.reset(outerBufferInfo.getBuffer(), outerBufferInfo.getStartOffset(), outerBufferInfo.getLength());
- int tupleCount = accessorOuter.getTupleCount();
- for (int i = 0; i < tupleCount; ++i) {
- final int[] ntFieldEndOffsets = missingTupleBuilder.getFieldEndOffsets();
- final byte[] ntByteArray = missingTupleBuilder.getByteArray();
- final int ntSize = missingTupleBuilder.getSize();
- FrameUtils.appendConcatToWriter(writer, appender, accessorOuter, i, ntFieldEndOffsets, ntByteArray, 0,
- ntSize);
+ private void appendMissing(int outerFrameMngrStartPos, int outerFrameTupleCount, IFrameWriter writer)
+ throws HyracksDataException {
+ int limit = outerFrameMngrStartPos + outerFrameTupleCount;
+ for (int outerTuplePos =
+ outerMatchLOJ.nextClearBit(outerFrameMngrStartPos); outerTuplePos < limit; outerTuplePos =
+ outerMatchLOJ.nextClearBit(outerTuplePos + 1)) {
+ int[] ntFieldEndOffsets = missingTupleBuilder.getFieldEndOffsets();
+ byte[] ntByteArray = missingTupleBuilder.getByteArray();
+ int ntSize = missingTupleBuilder.getSize();
+ int outerAccessorTupleIndex = outerTuplePos - outerFrameMngrStartPos;
+ FrameUtils.appendConcatToWriter(writer, appender, accessorOuter, outerAccessorTupleIndex, ntFieldEndOffsets,
+ ntByteArray, 0, ntSize);
}
}
@@ -203,22 +250,10 @@ public class NestedLoopJoin {
}
public void completeJoin(IFrameWriter writer) throws HyracksDataException {
- RunFileReader runFileReader = runFileWriter.createDeleteOnCloseReader();
try {
- runFileReader.open();
- if (runFileReader.nextFrame(innerBuffer)) {
- do {
- for (int i = 0; i < outerBufferMngr.getNumFrames(); i++) {
- blockJoin(outerBufferMngr.getFrame(i, tempInfo), innerBuffer.getBuffer(), writer);
- }
- } while (runFileReader.nextFrame(innerBuffer));
- } else if (isLeftOuter) {
- for (int i = 0; i < outerBufferMngr.getNumFrames(); i++) {
- appendMissing(outerBufferMngr.getFrame(i, tempInfo), writer);
- }
- }
+ multiBlockJoin(writer);
} finally {
- runFileReader.close();
+ runFileWriter.eraseClosed();
}
appender.write(writer, true);
}
@@ -226,8 +261,4 @@ public class NestedLoopJoin {
public void releaseMemory() throws HyracksDataException {
outerBufferMngr.reset();
}
-
- public void setIsReversed(boolean b) {
- this.isReversed = b;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
index 97f9c24..11d2c5a 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
@@ -795,11 +795,10 @@ public class OptimizedHybridHashJoinOperatorDescriptor extends AbstractOperatorD
// The nested loop join result is outer + inner. All the other operator is probe + build.
// Hence the reverse relation is different.
boolean isReversed = outerRd == buildRd && innerRd == probeRd;
- assert isLeftOuter ? !isReversed : true : "LeftOut Join can not reverse roles";
ITuplePairComparator nljComptorOuterInner = isReversed ? buildComp : probComp;
NestedLoopJoin nlj = new NestedLoopJoin(jobletCtx, new FrameTupleAccessor(outerRd),
- new FrameTupleAccessor(innerRd), memorySize, predEvaluator, isLeftOuter, nonMatchWriter);
- nlj.setIsReversed(isReversed);
+ new FrameTupleAccessor(innerRd), memorySize, predEvaluator, isLeftOuter, nonMatchWriter,
+ isReversed);
nlj.setComparator(nljComptorOuterInner);
IFrame cacheBuff = new VSizeFrame(jobletCtx);
[asterixdb] 21/23: Merge branch 'gerrit/mad-hatter'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 5daf5eaf2bca1617177a8d04d74ed25313951968
Merge: f2639a0 562fe20
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Mon Jun 28 15:05:13 2021 -0700
Merge branch 'gerrit/mad-hatter'
Change-Id: I7d5ea9a4967ee571e42ddf92d5f3bfba7f9cc911
.../optimizer/rules/PushGroupByThroughProduct.java | 17 ++++++-----
.../gby-cross-join/gby-cross-join.1.ddl.sqlpp | 34 ++++++++++++++++++++++
.../gby-cross-join/gby-cross-join.2.update.sqlpp | 34 ++++++++++++++++++++++
.../gby-cross-join/gby-cross-join.3.query.sqlpp | 25 ++++++++++++++++
.../group-by/gby-cross-join/gby-cross-join.3.adm | 2 ++
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 ++++
6 files changed, 110 insertions(+), 7 deletions(-)
[asterixdb] 06/23: [NO ISSUE][REP] Account for SSL socket data
transfer behavior
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 7b1a724b1653115c5fc4d46e44e1ced715957133
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Wed Jun 9 00:21:33 2021 +0300
[NO ISSUE][REP] Account for SSL socket data transfer behavior
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Account for the fact that a call to an SSLSocketChannel
read operation can return 0 read bytes even in a blocking
socket due to incomplete SSL packet.
Change-Id: I0e1e69cba7336e0cfca5def870ab16334ce8c19f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11064
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
(cherry picked from commit 94b1306e380dc923a964abf60900d55e545a4e4f)
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11843
---
.../replication/management/NetworkingUtil.java | 40 ++++++++++++++--------
1 file changed, 26 insertions(+), 14 deletions(-)
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
index d572ba2..7f6439c 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
@@ -34,9 +34,13 @@ import java.util.Enumeration;
import org.apache.hyracks.api.comm.NetworkAddress;
import org.apache.hyracks.api.network.ISocketChannel;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class NetworkingUtil {
+ private static final Logger LOGGER = LogManager.getLogger();
+
private NetworkingUtil() {
throw new AssertionError("This util class should not be initialized.");
}
@@ -56,25 +60,33 @@ public class NetworkingUtil {
public static void sendFile(FileChannel fileChannel, ISocketChannel socketChannel) throws IOException {
long pos = 0;
- long fileSize = fileChannel.size();
- long remainingBytes = fileSize;
- long transferredBytes = 0;
-
- while ((transferredBytes += fileChannel.transferTo(pos, remainingBytes, socketChannel)) < fileSize) {
- pos += transferredBytes;
- remainingBytes -= transferredBytes;
+ long remainingBytes = fileChannel.size();
+ try {
+ while (remainingBytes > 0) {
+ long sentBytes = fileChannel.transferTo(pos, remainingBytes, socketChannel);
+ pos += sentBytes;
+ remainingBytes -= sentBytes;
+ }
+ socketChannel.getSocketChannel().socket().getOutputStream().flush();
+ } catch (Exception e) {
+ LOGGER.info("failed to send file; file size {}, pos {}, remainingBytes {}", fileChannel.size(), pos,
+ remainingBytes);
}
- socketChannel.getSocketChannel().socket().getOutputStream().flush();
}
public static void downloadFile(FileChannel fileChannel, ISocketChannel socketChannel) throws IOException {
+ long remainingBytes = fileChannel.size();
long pos = 0;
- long fileSize = fileChannel.size();
- long count = fileSize;
- long numTransferred = 0;
- while ((numTransferred += fileChannel.transferFrom(socketChannel, pos, count)) < fileSize) {
- pos += numTransferred;
- count -= numTransferred;
+ try {
+ while (remainingBytes > 0) {
+ long readBytes = fileChannel.transferFrom(socketChannel, pos, remainingBytes);
+ pos += readBytes;
+ remainingBytes -= readBytes;
+ }
+ } catch (Exception e) {
+ LOGGER.info("failed to download file; file size {}, pos {}, remainingBytes {}", fileChannel.size(), pos,
+ remainingBytes);
+ throw e;
}
}
[asterixdb] 18/23: [NO ISSUE] Update maven-antrun-plugin 1.8 ->
3.0.0
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 6db8b8e3b158742235b645036bebb38825ce5443
Author: Michael Blow <mb...@apache.org>
AuthorDate: Mon Jun 28 06:23:53 2021 -0400
[NO ISSUE] Update maven-antrun-plugin 1.8 -> 3.0.0
Change-Id: Ie014329b3fba3b4c7cb0c094b1d9eb1ae9133486
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12103
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
asterixdb/pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index ecda2c1..c56c43f 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -285,7 +285,7 @@
<condition property="skipPomEnforcement">
<not><equals arg1="${project.packaging}" arg2="pom"/></not>
</condition>
- <echo message="will enforce non-existence of test source dir due to ${project.packaging} packaging" unless:set="skipPomEnforcement"/>
+ <echo level="info" message="will enforce non-existence of test source dir due to ${project.packaging} packaging" unless:set="skipPomEnforcement"/>
</target>
</configuration>
</execution>
@@ -566,7 +566,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
- <version>1.8</version>
+ <version>3.0.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
[asterixdb] 09/23: [NO ISSUE][EXT]: Retry upon failure for S3
retryable errors
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 4d2a73fcdfa07295b7ef90ddcf01a2e3c0777afc
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Wed Jun 9 19:59:23 2021 +0300
[NO ISSUE][EXT]: Retry upon failure for S3 retryable errors
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Retry upon failure for S3 retryable errors.
Change-Id: I639fd7d43c2a6c28b3cc4247bf9ac5d3a23a387e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11883
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
.../input/record/reader/aws/AwsS3InputStream.java | 55 ++++++--
.../record/reader/aws/AwsS3InputStreamFactory.java | 4 +-
.../external/util/ExternalDataConstants.java | 9 ++
.../asterix/external/util/ExternalDataUtils.java | 3 +-
.../asterix/external/input/awss3/AwsS3Test.java | 142 +++++++++++++++++++++
5 files changed, 199 insertions(+), 14 deletions(-)
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
index 8bd7a51..4d5288c 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
@@ -23,6 +23,7 @@ import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3;
import java.io.IOException;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import java.util.zip.GZIPInputStream;
import org.apache.asterix.common.exceptions.CompilationException;
@@ -41,6 +42,7 @@ import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.model.S3Exception;
public class AwsS3InputStream extends AbstractMultipleInputStream {
@@ -51,6 +53,7 @@ public class AwsS3InputStream extends AbstractMultipleInputStream {
private final int bufferSize;
private final S3Client s3Client;
+ private static final int MAX_RETRIES = 5; // We will retry 5 times in case of internal error from AWS S3 service
// File fields
private final List<String> filePaths;
@@ -84,17 +87,10 @@ public class AwsS3InputStream extends AbstractMultipleInputStream {
GetObjectRequest.Builder getObjectBuilder = GetObjectRequest.builder();
GetObjectRequest getObjectRequest = getObjectBuilder.bucket(bucket).key(fileName).build();
- // Have a reference to the S3 stream to ensure that if GZipInputStream causes an IOException because of reading
- // the header, then the S3 stream gets closed in the close method
- try {
- in = s3Client.getObject(getObjectRequest);
- } catch (NoSuchKeyException ex) {
- LOGGER.debug(() -> "Key " + LogRedactionUtil.userData(getObjectRequest.key()) + " was not found in bucket "
- + getObjectRequest.bucket());
- nextFileIndex++;
+ boolean isAvailableStream = doGetInputStream(getObjectRequest);
+ nextFileIndex++;
+ if (!isAvailableStream) {
return advance();
- } catch (SdkException ex) {
- throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
}
// Use gzip stream if needed
@@ -103,13 +99,50 @@ public class AwsS3InputStream extends AbstractMultipleInputStream {
}
// Current file ready, point to the next file
- nextFileIndex++;
if (notificationHandler != null) {
notificationHandler.notifyNewSource();
}
return true;
}
+ /**
+ * Get the input stream. If an error is encountered, depending on the error code, a retry might be favorable.
+ *
+ * @return true
+ */
+ private boolean doGetInputStream(GetObjectRequest request) throws RuntimeDataException {
+ int retries = 0;
+ while (retries < MAX_RETRIES) {
+ try {
+ in = s3Client.getObject(request);
+ break;
+ } catch (NoSuchKeyException ex) {
+ LOGGER.debug(() -> "Key " + LogRedactionUtil.userData(request.key()) + " was not found in bucket "
+ + request.bucket());
+ return false;
+ } catch (S3Exception ex) {
+ if (!shouldRetry(ex.awsErrorDetails().errorCode(), retries++)) {
+ throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
+ }
+ LOGGER.debug(() -> "S3 retryable error: " + LogRedactionUtil.userData(ex.getMessage()));
+
+ // Backoff for 1 sec for the first 2 retries, and 2 seconds from there onward
+ try {
+ Thread.sleep(TimeUnit.SECONDS.toMillis(retries < 3 ? 1 : 2));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ } catch (SdkException ex) {
+ throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
+ }
+ }
+ return true;
+ }
+
+ private boolean shouldRetry(String errorCode, int currentRetry) {
+ return currentRetry < MAX_RETRIES && AwsS3.isRetryableError(errorCode);
+ }
+
private S3Client buildAwsS3Client(Map<String, String> configuration) throws HyracksDataException {
try {
return ExternalDataUtils.AwsS3.buildAwsS3Client(configuration);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
index 0bc4c40..715c5df 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
@@ -136,7 +136,7 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
// New API is not implemented, try falling back to old API
try {
// For error code, see https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
- if (ex.awsErrorDetails().errorCode().equals("NotImplemented")) {
+ if (ex.awsErrorDetails().errorCode().equals(AwsS3.ERROR_METHOD_NOT_IMPLEMENTED)) {
filesOnly = oldApiListS3Objects(s3Client, container, matchersList, p);
} else {
throw ex;
@@ -312,7 +312,7 @@ public class AwsS3InputStreamFactory implements IInputStreamFactory {
return smallest;
}
- private static class PartitionWorkLoadBasedOnSize implements Serializable {
+ public static class PartitionWorkLoadBasedOnSize implements Serializable {
private static final long serialVersionUID = 1L;
private final List<String> filePaths = new ArrayList<>();
private long totalSize = 0;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 1911083..0f4117b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -291,5 +291,14 @@ public class ExternalDataConstants {
public static final String SECRET_ACCESS_KEY_FIELD_NAME = "secretAccessKey";
public static final String CONTAINER_NAME_FIELD_NAME = "container";
public static final String SERVICE_END_POINT_FIELD_NAME = "serviceEndpoint";
+
+ // AWS S3 specific error codes
+ public static final String ERROR_INTERNAL_ERROR = "InternalError";
+ public static final String ERROR_SLOW_DOWN = "SlowDown";
+ public static final String ERROR_METHOD_NOT_IMPLEMENTED = "NotImplemented";
+
+ public static boolean isRetryableError(String errorCode) {
+ return errorCode.equals(ERROR_INTERNAL_ERROR) || errorCode.equals(ERROR_SLOW_DOWN);
+ }
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 363ec74..a6b8b47 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -710,7 +710,8 @@ public class ExternalDataUtils {
// Method not implemented, try falling back to old API
try {
// For error code, see https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
- if (ex.awsErrorDetails().errorCode().equals("NotImplemented")) {
+ if (ex.awsErrorDetails().errorCode()
+ .equals(ExternalDataConstants.AwsS3.ERROR_METHOD_NOT_IMPLEMENTED)) {
useOldApi = true;
response = isBucketEmpty(s3Client, container, prefix, true);
} else {
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/awss3/AwsS3Test.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/awss3/AwsS3Test.java
new file mode 100644
index 0000000..f0a2223
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/awss3/AwsS3Test.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.awss3;
+
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_INTERNAL_ERROR;
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_SLOW_DOWN;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.external.input.record.reader.aws.AwsS3InputStream;
+import org.apache.asterix.external.input.record.reader.aws.AwsS3InputStreamFactory;
+import org.apache.hyracks.api.exceptions.IFormattedException;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+public class AwsS3Test {
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testWorkloadDistribution() throws Exception {
+ AwsS3InputStreamFactory factory = new AwsS3InputStreamFactory();
+
+ List<S3Object> s3Objects = new ArrayList<>();
+ final int partitionsCount = 3;
+
+ // Create S3 objects, 9 objects, on 3 partitions, they should be 600 total size on each partition
+ S3Object.Builder builder = S3Object.builder();
+ s3Objects.add(builder.key("1.json").size(100L).build());
+ s3Objects.add(builder.key("2.json").size(100L).build());
+ s3Objects.add(builder.key("3.json").size(100L).build());
+ s3Objects.add(builder.key("4.json").size(200L).build());
+ s3Objects.add(builder.key("5.json").size(200L).build());
+ s3Objects.add(builder.key("6.json").size(200L).build());
+ s3Objects.add(builder.key("7.json").size(300L).build());
+ s3Objects.add(builder.key("8.json").size(300L).build());
+ s3Objects.add(builder.key("9.json").size(300L).build());
+
+ // invoke the distributeWorkLoad method
+ Method distributeWorkloadMethod =
+ AwsS3InputStreamFactory.class.getDeclaredMethod("distributeWorkLoad", List.class, int.class);
+ distributeWorkloadMethod.setAccessible(true);
+ distributeWorkloadMethod.invoke(factory, s3Objects, partitionsCount);
+
+ // get the partitionWorkLoadsBasedOnSize field and verify the result
+ Field distributeWorkloadField = AwsS3InputStreamFactory.class.getDeclaredField("partitionWorkLoadsBasedOnSize");
+ distributeWorkloadField.setAccessible(true);
+ List<AwsS3InputStreamFactory.PartitionWorkLoadBasedOnSize> workloads =
+ (List<AwsS3InputStreamFactory.PartitionWorkLoadBasedOnSize>) distributeWorkloadField.get(factory);
+
+ for (AwsS3InputStreamFactory.PartitionWorkLoadBasedOnSize workload : workloads) {
+ Assert.assertEquals(workload.getTotalSize(), 600);
+ }
+ }
+
+ @Test
+ public void s3InternalError() throws Exception {
+ // S3Client mock
+ S3Client s3ClientMock = Mockito.mock(S3Client.class);
+
+ // Prepare S3Exception with internal error code
+ AwsErrorDetails errorDetails = AwsErrorDetails.builder().errorCode(ERROR_INTERNAL_ERROR)
+ .errorMessage("Internal Error from AWS").build();
+ S3Exception internalErrorEx = (S3Exception) S3Exception.builder().awsErrorDetails(errorDetails).build();
+ Mockito.when(s3ClientMock.getObject(GetObjectRequest.builder().build())).thenThrow(internalErrorEx);
+
+ // Set S3Client mock
+ AwsS3InputStream inputStreamMock = Mockito.mock(AwsS3InputStream.class);
+ Field s3ClientField = AwsS3InputStream.class.getDeclaredField("s3Client");
+ s3ClientField.setAccessible(true);
+ s3ClientField.set(inputStreamMock, s3ClientMock);
+
+ // doGetInputStream method
+ Method doGetInputStreamMethod =
+ AwsS3InputStream.class.getDeclaredMethod("doGetInputStream", GetObjectRequest.class);
+ doGetInputStreamMethod.setAccessible(true);
+
+ try {
+ doGetInputStreamMethod.invoke(inputStreamMock, GetObjectRequest.builder().build());
+ } catch (Exception ex) {
+ Assert.assertTrue("Not internal error", ex.getCause() instanceof IFormattedException
+ && ex.getCause().toString().contains("ASX1108: External source error. Internal Error from AWS"));
+ }
+ }
+
+ @Test
+ public void s3SlowDown() throws Exception {
+ // S3Client mock
+ S3Client s3ClientMock = Mockito.mock(S3Client.class);
+
+ // Prepare S3Exception with slow down error code
+ AwsErrorDetails errorDetails =
+ AwsErrorDetails.builder().errorCode(ERROR_SLOW_DOWN).errorMessage("SlowDown Error from AWS").build();
+ S3Exception slowDownEx = (S3Exception) S3Exception.builder().awsErrorDetails(errorDetails).build();
+ Mockito.when(s3ClientMock.getObject(GetObjectRequest.builder().build())).thenThrow(slowDownEx);
+
+ // Set S3Client mock
+ AwsS3InputStream inputStreamMock = Mockito.mock(AwsS3InputStream.class);
+
+ // Set S3Client
+ Field s3ClientField = AwsS3InputStream.class.getDeclaredField("s3Client");
+ s3ClientField.setAccessible(true);
+ s3ClientField.set(inputStreamMock, s3ClientMock);
+
+ // doGetInputStream method
+ Method doGetInputStreamMethod =
+ AwsS3InputStream.class.getDeclaredMethod("doGetInputStream", GetObjectRequest.class);
+ doGetInputStreamMethod.setAccessible(true);
+
+ try {
+ doGetInputStreamMethod.invoke(inputStreamMock, GetObjectRequest.builder().build());
+ } catch (Exception ex) {
+ Assert.assertTrue("Not SlowDown error", ex.getCause() instanceof IFormattedException
+ && ex.getCause().toString().contains("ASX1108: External source error. SlowDown Error from AWS"));
+ }
+ }
+}
[asterixdb] 08/23: [ASTERIXDB-2905] Error when window function used
in subquery
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 8f21c056e205b5479d95302387c0f6e522d403bf
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Tue May 25 10:43:01 2021 -0700
[ASTERIXDB-2905] Error when window function used in subquery
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Fix ambiguous alias failure when window function
is used by subquery
- Add testcase
Change-Id: I4011af7e46042b9cb0279d0f182deede8bc178cf
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11624
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
---
.../window/misc_01/misc_01.10.query.sqlpp | 34 +++++++++++++++
.../results/window/misc_01/misc_01.10.adm | 4 ++
.../window/misc_01/misc_01.10.ast | 50 ++++++++++++++++++++++
.../SqlppWindowAggregationSugarVisitor.java | 7 ++-
4 files changed, 93 insertions(+), 2 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/window/misc_01/misc_01.10.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/window/misc_01/misc_01.10.query.sqlpp
new file mode 100644
index 0000000..57545a6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/window/misc_01/misc_01.10.query.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test multiple window functions in the same statement
+ * Expected Res : SUCCESS
+ */
+
+use test;
+
+select c2,
+ (
+ select nth_value(c2, 3)
+ over (partition by one order by c2 range between unbounded preceding and unbounded following)
+ as nth
+ from t1 x
+ ) as q1
+from t1 y
+order by c2;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/window/misc_01/misc_01.10.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/window/misc_01/misc_01.10.adm
new file mode 100644
index 0000000..8e98bdc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/window/misc_01/misc_01.10.adm
@@ -0,0 +1,4 @@
+{ "q1": [ { "nth": 3 }, { "nth": 3 }, { "nth": 3 }, { "nth": 3 } ], "c2": 1 }
+{ "q1": [ { "nth": 3 }, { "nth": 3 }, { "nth": 3 }, { "nth": 3 } ], "c2": 2 }
+{ "q1": [ { "nth": 3 }, { "nth": 3 }, { "nth": 3 }, { "nth": 3 } ], "c2": 3 }
+{ "q1": [ { "nth": 3 }, { "nth": 3 }, { "nth": 3 }, { "nth": 3 } ], "c2": 4 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/window/misc_01/misc_01.10.ast b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/window/misc_01/misc_01.10.ast
new file mode 100644
index 0000000..368f261
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/window/misc_01/misc_01.10.ast
@@ -0,0 +1,50 @@
+DataverseUse test
+Query:
+SELECT [
+FieldAccessor [
+ Variable [ Name=$y ]
+ Field=c2
+]
+c2
+(
+ SELECT [
+ WINDOW test.nth_value@2[
+ Variable [ Name=$c2 ]
+ LiteralExpr [LONG] [3]
+ ]
+ OVER (
+ PARTITION BY
+ FieldAccessor [
+ Variable [ Name=$x ]
+ Field=one
+ ]
+ ORDER BY
+ FieldAccessor [
+ Variable [ Name=$x ]
+ Field=c2
+ ]
+ ASC
+ range between unbounded preceding and unbounded following exclude no others
+ )
+ nth
+ ]
+ FROM [ FunctionCall asterix.dataset@1[
+ LiteralExpr [STRING] [test.t1]
+ ]
+ AS Variable [ Name=$x ]
+ ]
+)
+q1
+]
+FROM [ FunctionCall asterix.dataset@1[
+ LiteralExpr [STRING] [test.t1]
+ ]
+ AS Variable [ Name=$y ]
+]
+Orderby
+ FieldAccessor [
+ Variable [ Name=$y ]
+ Field=c2
+ ]
+ ASC
+
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppWindowAggregationSugarVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppWindowAggregationSugarVisitor.java
index a5f43b6..bed2fed 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppWindowAggregationSugarVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppWindowAggregationSugarVisitor.java
@@ -105,8 +105,11 @@ public class SqlppWindowAggregationSugarVisitor extends AbstractSqlppExpressionS
Map<VariableExpr, Set<? extends Scope.SymbolAnnotation>> liveAnnotatedVars =
scopeChecker.getCurrentScope().getLiveVariables();
Set<VariableExpr> liveVars = liveAnnotatedVars.keySet();
- Set<VariableExpr> liveContextVars = Scope.findVariablesAnnotatedBy(liveVars,
- SqlppVariableAnnotation.CONTEXT_VARIABLE, liveAnnotatedVars, winExpr.getSourceLocation());
+
+ Map<VariableExpr, Set<? extends Scope.SymbolAnnotation>> localAnnotatedVars =
+ scopeChecker.getCurrentScope().getLiveVariables(scopeChecker.getPrecedingScope());
+ Set<VariableExpr> liveContextVars = Scope.findVariablesAnnotatedBy(localAnnotatedVars.keySet(),
+ SqlppVariableAnnotation.CONTEXT_VARIABLE, localAnnotatedVars, winExpr.getSourceLocation());
List<Pair<Expression, Identifier>> winFieldList = winExpr.getWindowFieldList();
Map<VariableExpr, Identifier> winVarFieldMap =
[asterixdb] 13/23: [ASTERIXDB-2886][COMP] Fix
RemoveRedundantVariablesRule
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 32fcc2ae6e30c601e6408fa60a59d1eb2ad38c93
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Tue Apr 20 14:38:53 2021 -0700
[ASTERIXDB-2886][COMP] Fix RemoveRedundantVariablesRule
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Change RemoveRedundantVariablesRule to
operate on the whole plan at once instead
of working incrementally on each operator
Change-Id: Ie948372c53bf42687ffd0ac37eff39bddf7395bc
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11863
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../optimizerts/queries/fj-dblp-csx-hybrid.aql | 69 -------------
.../optimizerts/queries/fj-dblp-csx-selflink.aql | 66 -------------
.../optimizerts/queries/fj-dblp-csx-simple.aql | 54 ----------
.../optimizerts/queries/fj-dblp-csx-star.aql | 69 -------------
.../ngram-fuzzyeq-jaccard_01.aql | 60 ------------
.../inverted-index-join/ngram-jaccard-check_01.aql | 58 -----------
.../inverted-index-join/ngram-jaccard_01.aql | 58 -----------
.../word-fuzzyeq-jaccard_01.aql | 59 -----------
.../inverted-index-join/word-jaccard-check_01.aql | 57 -----------
.../inverted-index-join/word-jaccard_01.aql | 58 -----------
...n-probe-pidx-with-join-jaccard-check-idx_01.aql | 72 --------------
.../ngram-fuzzyeq-jaccard_01.aql | 68 -------------
.../inverted-index-join/ngram-jaccard-check_01.aql | 66 -------------
.../inverted-index-join/ngram-jaccard-inline.aql | 54 ----------
.../inverted-index-join/ngram-jaccard_01.aql | 66 -------------
.../word-fuzzyeq-jaccard_01.aql | 67 -------------
.../word-jaccard-check-after-btree-access.aql | 73 --------------
.../inverted-index-join/word-jaccard-check_01.aql | 65 ------------
.../inverted-index-join/word-jaccard-inline.aql | 53 ----------
.../inverted-index-join/word-jaccard_01.aql | 66 -------------
.../ngram-fuzzyeq-jaccard_01.aql | 67 -------------
.../ngram-fuzzyeq-jaccard_02.aql | 67 -------------
.../ngram-fuzzyeq-jaccard_03.aql | 53 ----------
.../ngram-fuzzyeq-jaccard_04.aql | 68 -------------
.../inverted-index-join/ngram-jaccard-check_01.aql | 65 ------------
.../inverted-index-join/ngram-jaccard-check_02.aql | 65 ------------
.../inverted-index-join/ngram-jaccard-check_03.aql | 51 ----------
.../inverted-index-join/ngram-jaccard-check_04.aql | 66 -------------
.../inverted-index-join/ngram-jaccard-inline.aql | 53 ----------
.../inverted-index-join/ngram-jaccard_01.aql | 65 ------------
.../inverted-index-join/ngram-jaccard_02.aql | 65 ------------
.../inverted-index-join/ngram-jaccard_03.aql | 51 ----------
.../inverted-index-join/ngram-jaccard_04.aql | 66 -------------
.../word-fuzzyeq-jaccard_01.aql | 66 -------------
.../word-fuzzyeq-jaccard_02.aql | 66 -------------
.../word-fuzzyeq-jaccard_03.aql | 51 ----------
.../word-fuzzyeq-jaccard_04.aql | 67 -------------
.../word-jaccard-check-after-btree-access.aql | 72 --------------
.../inverted-index-join/word-jaccard-check_01.aql | 64 ------------
.../inverted-index-join/word-jaccard-check_02.aql | 64 ------------
.../inverted-index-join/word-jaccard-check_03.aql | 50 ----------
.../inverted-index-join/word-jaccard-check_04.aql | 65 ------------
.../inverted-index-join/word-jaccard-inline.aql | 52 ----------
.../inverted-index-join/word-jaccard_01.aql | 65 ------------
.../inverted-index-join/word-jaccard_02.aql | 65 ------------
.../inverted-index-join/word-jaccard_03.aql | 51 ----------
.../inverted-index-join/word-jaccard_04.aql | 66 -------------
.../ngram-fuzzyeq-jaccard_01.aql | 59 -----------
.../ngram-fuzzyeq-jaccard_02.aql | 59 -----------
.../ngram-fuzzyeq-jaccard_03.aql | 49 ---------
.../ngram-fuzzyeq-jaccard_04.aql | 60 ------------
.../inverted-index-join/ngram-jaccard-check_01.aql | 57 -----------
.../inverted-index-join/ngram-jaccard-check_02.aql | 57 -----------
.../inverted-index-join/ngram-jaccard-check_03.aql | 47 ---------
.../inverted-index-join/ngram-jaccard-check_04.aql | 58 -----------
.../ngram-jaccard-check_inline_03.aql | 48 ---------
.../inverted-index-join/ngram-jaccard_01.aql | 57 -----------
.../inverted-index-join/ngram-jaccard_02.aql | 57 -----------
.../inverted-index-join/ngram-jaccard_03.aql | 47 ---------
.../inverted-index-join/ngram-jaccard_04.aql | 58 -----------
.../ngram-jaccard_inline_03.aql | 48 ---------
.../word-fuzzyeq-jaccard_01.aql | 58 -----------
.../word-fuzzyeq-jaccard_02.aql | 58 -----------
.../word-fuzzyeq-jaccard_03.aql | 48 ---------
.../word-fuzzyeq-jaccard_04.aql | 59 -----------
.../word-jaccard-check-after-btree-access.aql | 68 -------------
.../inverted-index-join/word-jaccard-check_01.aql | 56 -----------
.../inverted-index-join/word-jaccard-check_02.aql | 56 -----------
.../inverted-index-join/word-jaccard-check_03.aql | 46 ---------
.../inverted-index-join/word-jaccard-check_04.aql | 57 -----------
.../word-jaccard-check_inline_03.aql | 47 ---------
.../inverted-index-join/word-jaccard_01.aql | 57 -----------
.../inverted-index-join/word-jaccard_02.aql | 57 -----------
.../inverted-index-join/word-jaccard_03.aql | 46 ---------
.../inverted-index-join/word-jaccard_04.aql | 58 -----------
.../inverted-index-join/word-jaccard_inline_03.aql | 47 ---------
.../jaccard-similarity-join-dual-order.aql | 109 ---------------------
.../optimizerts/queries/unnest_list_in_subplan.aql | 55 -----------
.../ngram-jaccard-inline.plan | 8 +-
.../word-jaccard-inline.plan | 8 +-
.../results/inverted-index-join/issue741.plan | 8 +-
...-probe-pidx-with-join-jaccard-check-idx_01.plan | 8 +-
...obe-pidx-with-join-jaccard-check-idx_01_ps.plan | 16 +--
.../ngram-fuzzyeq-jaccard_02.plan | 8 +-
.../ngram-fuzzyeq-jaccard_03.plan | 8 +-
.../ngram-jaccard-check_02.plan | 8 +-
.../ngram-jaccard-check_03.plan | 8 +-
.../ngram-jaccard-check_04.plan | 8 +-
.../inverted-index-join/ngram-jaccard_02.plan | 8 +-
.../inverted-index-join/ngram-jaccard_03.plan | 8 +-
.../inverted-index-join/ngram-jaccard_04.plan | 8 +-
.../word-fuzzyeq-jaccard_02.plan | 8 +-
.../word-fuzzyeq-jaccard_03.plan | 8 +-
.../word-jaccard-check-after-btree-access.plan | 8 +-
.../inverted-index-join/word-jaccard-check_02.plan | 8 +-
.../inverted-index-join/word-jaccard-check_03.plan | 8 +-
.../inverted-index-join/word-jaccard-check_04.plan | 8 +-
.../inverted-index-join/word-jaccard_02.plan | 8 +-
.../inverted-index-join/word-jaccard_03.plan | 8 +-
.../inverted-index-join/word-jaccard_04.plan | 8 +-
...obe-pidx-with-join-jaccard-check-idx_01_ps.plan | 16 +--
.../query-ASTERIXDB-2886.1.ddl.sqlpp | 43 ++++++++
.../query-ASTERIXDB-2886.2.ddl.sqlpp | 43 ++++++++
.../query-ASTERIXDB-2886.3.query.sqlpp | 30 ++++++
.../query-ASTERIXDB-2886.3.adm | 4 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 +
.../rules/RemoveRedundantVariablesRule.java | 94 ++++++++++--------
107 files changed, 278 insertions(+), 4808 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/fj-dblp-csx-hybrid.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/fj-dblp-csx-hybrid.aql
deleted file mode 100644
index a753907..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/fj-dblp-csx-hybrid.aql
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Multiple fuzzy join on three datasets, with a star join condition.
- * Each star join is composed of a linked join condition to propagate
- * the fuzzy join results.
- * Success : Yes
- */
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPNestedType as closed {
- id: int64,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested: DBLPNestedType
-}
-
-create type CSXNestedType as closed {
- id: int64,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as closed {
- nested: CSXNestedType
-}
-
-create dataset DBLPOpen(DBLPNestedType) primary key id;
-
-create dataset DBLP(DBLPType) primary key nested.id;
-create dataset CSX(CSXType) primary key nested.id;
-
-set import-private-functions 'true';
-set simthreshold "0.5f";
-for $p in dataset DBLP
-for $p1 in dataset CSX
-for $p2 in dataset DBLPOpen
-for $p3 in dataset CSX
-for $p4 in dataset DBLPOpen
-where gram-tokens($p.nested.title, 3, false) ~= gram-tokens($p1.nested.title, 3, false)
-and word-tokens($p1.nested.title) ~= word-tokens($p2.title)
-and word-tokens($p.authors) ~= word-tokens($p3.nested.authors)
-and gram-tokens($p3.nested.misc, 3, false) ~= gram-tokens($p4.misc, 3, false)
-return { "pid": $p.nested.id, "p1id": $p1.nested.id, "p2id": $p2.id, "p3id": $p3.nested.id, "p4id": $p4.id }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/fj-dblp-csx-selflink.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/fj-dblp-csx-selflink.aql
deleted file mode 100644
index 1b3f930..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/fj-dblp-csx-selflink.aql
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy join on two datasets, with a series of link join conditions.
- * The dataset DBLP will fuzzy join with CSX and propagate the results
- * with a series of linked join conditions on CSX.
- * Success : Yes
- */
-drop dataverse test if exists;
-
-create dataverse test;
-
-use dataverse test;
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as open {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create nodegroup group1 if not exists on asterix_nc1, asterix_nc2;
-
-create dataset DBLP(DBLPType) primary key id on group1;
-create dataset CSX(CSXType) primary key id on group1;
-
-write output to asterix_nc1:'rttest/test.adm';
-
-use dataverse test;
-set import-private-functions 'true';
-set simthreshold "0.5f";
-for $p in dataset DBLP
-for $p1 in dataset CSX
-for $p2 in dataset CSX
-for $p3 in dataset CSX
-for $p4 in dataset CSX
-where word-tokens($p.title) ~= word-tokens($p1.title)
-and word-tokens($p2.title) ~= word-tokens($p3.title)
-and gram-tokens($p1.authors, 3, false) ~= gram-tokens($p2.authors, 2, false)
-and gram-tokens($p3.misc, 3, false) ~= gram-tokens($p4.misc, 3, false)
-return { "pid":$p.id, "p1id":$p1.id, "p2id": $p2.id, "p3id": $p3.id, "p4id": $p4.id }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/fj-dblp-csx-simple.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/fj-dblp-csx-simple.aql
deleted file mode 100644
index b0beb4d..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/fj-dblp-csx-simple.aql
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
- /*
- * Description : Two-way fuzzy join on DBLP and CSX with a fuzzy join condition on word tokens of their titles.
- * Success : Yes
- */
-drop dataverse fj-dblp-csx if exists;
-
-create dataverse fj-dblp-csx;
-
-use dataverse fj-dblp-csx;
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as open {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create nodegroup group1 if not exists on asterix_nc1, asterix_nc2;
-
-create dataset DBLP(DBLPType) primary key id on group1;
-create dataset CSX(CSXType) primary key id on group1;
-
-write output to asterix_nc1:'rttest/fj-dblp-csx.adm';
-for $s in dataset('DBLP')
-for $t in dataset('CSX')
-where word-tokens($s.title) ~= word-tokens($t.title)
-return {"sid": $s.id, "tid": $t.id}
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/fj-dblp-csx-star.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/fj-dblp-csx-star.aql
deleted file mode 100644
index df0cb86..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/fj-dblp-csx-star.aql
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy join on three datasets, with a set of star join conditions.
- * The CSX and DBLP are used twice and will be propagated onto the output with their ids.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPNestedType as closed {
- id: int64,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested: DBLPNestedType
-}
-
-create type CSXNestedType as closed {
- id: int64,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as closed {
- nested: CSXNestedType
-}
-
-create dataset DBLPOpen(DBLPNestedType) primary key id;
-
-create dataset DBLP(DBLPType) primary key nested.id;
-create dataset CSX(CSXType) primary key nested.id;
-
-set import-private-functions 'true';
-set simthreshold "0.5f";
-for $p in dataset DBLP
-for $p1 in dataset CSX
-for $p2 in dataset DBLPOpen
-for $p3 in dataset CSX
-for $p4 in dataset DBLPOpen
-where word-tokens($p.nested.title) ~= word-tokens($p1.nested.title)
-and word-tokens($p.nested.authors) ~= word-tokens($p3.nested.authors)
-and word-tokens($p.nested.title) ~= word-tokens($p2.title)
-and gram-tokens($p.nested.title, 3, false) ~= gram-tokens($p4.title, 3, false)
-return { "pid": $p.nested.id, "p1id": $p1.nested.id, "p2id": $p2.id, "p3id": $p3.nested.id, "p4id": $p4.id }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/ngram-fuzzyeq-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/ngram-fuzzyeq-jaccard_01.aql
deleted file mode 100644
index a9aa579..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/ngram-fuzzyeq-jaccard_01.aql
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index ngram_index on DBLP(title) type ngram(3);
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where gram-tokens($a.title, 3, false) ~= gram-tokens($b.title, 3, false) and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/ngram-jaccard-check_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/ngram-jaccard-check_01.aql
deleted file mode 100644
index 07e6c18..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/ngram-jaccard-check_01.aql
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard-check function of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index ngram_index on DBLP(title) type ngram(3);
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard-check_01.adm";
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where similarity-jaccard-check(gram-tokens($a.title, 3, false), gram-tokens($b.title, 3, false), 0.5f)[0]
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/ngram-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/ngram-jaccard_01.aql
deleted file mode 100644
index a68d600..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/ngram-jaccard_01.aql
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index ngram_index on DBLP(title) type ngram(3);
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard_01.adm";
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where similarity-jaccard(gram-tokens($a.title, 3, false), gram-tokens($b.title, 3, false)) >= 0.5f
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/word-fuzzyeq-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/word-fuzzyeq-jaccard_01.aql
deleted file mode 100644
index 29bd3d0..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/word-fuzzyeq-jaccard_01.aql
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index keyword_index on DBLP(title) type keyword;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where word-tokens($a.title) ~= word-tokens($b.title) and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/word-jaccard-check_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/word-jaccard-check_01.aql
deleted file mode 100644
index 989f250..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/word-jaccard-check_01.aql
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard-check function of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index keyword_index on DBLP(title) type keyword;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard-check_01.adm";
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where similarity-jaccard-check(word-tokens($a.title), word-tokens($b.title), 0.5f)[0]
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/word-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/word-jaccard_01.aql
deleted file mode 100644
index e2c373a..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join/word-jaccard_01.aql
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index keyword_index on DBLP(title) type keyword;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard_01.adm";
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where similarity-jaccard(word-tokens($a.title), word-tokens($b.title)) >= 0.5f
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
-
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.aql
deleted file mode 100644
index 128b068..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.aql
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
- * Issue : 730, 741
- * Expected Res : Success
- * Date : 8th May 2014
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type TwitterUserType as closed {
- screen-name: string,
- lang: string,
- friends-count: int32,
- statuses-count: int32,
- name: string,
- followers-count: int32
-}
-
-create type TweetMessageNestedType as closed {
- tweetid: int64,
- user: TwitterUserType,
- sender-location: point,
- send-time: datetime,
- referred-topics: {{ string }},
- message-text: string,
- countA: int32,
- countB: int32
-}
-
-create type TweetMessageType as closed {
- nested: TweetMessageNestedType
-}
-
-create dataset TweetMessages(TweetMessageType)
-primary key nested.tweetid;
-
-create index topicKeywordIx on TweetMessages(nested.referred-topics) type keyword;
-
-write output to asterix_nc1:"rttest/inverted-index-join_leftouterjoin-probe-pidx-with-join-jaccard-check_idx_01.adm";
-
-for $t1 in dataset('TweetMessages')
-where $t1.nested.tweetid > int64("240")
-order by $t1.nested.tweetid
-return {
- "tweet": {"id": $t1.nested.tweetid, "topics" : $t1.nested.referred-topics} ,
- "similar-tweets": for $t2 in dataset('TweetMessages')
- let $sim := similarity-jaccard-check($t1.nested.referred-topics, $t2.nested.referred-topics, 0.5f)
- where $sim[0] and
- $t2.nested.tweetid != $t1.nested.tweetid
- order by $t2.nested.tweetid
- return {"id": $t2.nested.tweetid, "topics" : $t2.nested.referred-topics}
-};
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/ngram-fuzzyeq-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/ngram-fuzzyeq-jaccard_01.aql
deleted file mode 100644
index 56e0094..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/ngram-fuzzyeq-jaccard_01.aql
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index ngram_index on CSX(nested.title) type ngram(3);
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where gram-tokens($a.nested.title, 3, false) ~= gram-tokens($b.nested.title, 3, false) and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/ngram-jaccard-check_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/ngram-jaccard-check_01.aql
deleted file mode 100644
index 482e8c9..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/ngram-jaccard-check_01.aql
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard-check function of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index ngram_index on CSX(nested.title) type ngram(3);
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard-check_01.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where similarity-jaccard-check(gram-tokens($a.nested.title, 3, false), gram-tokens($b.nested.title, 3, false), 0.5f)[0]
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/ngram-jaccard-inline.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/ngram-jaccard-inline.aql
deleted file mode 100644
index 7448a06..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/ngram-jaccard-inline.aql
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, DBLP, based on the similarity-jaccard function of its titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
- * We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create index ngram_index on DBLP(nested.title) type ngram(3);
-
-write output to asterix_nc1:"rttest/inverted-index-join-noeqjoin_ngram-jaccard-inline.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-let $jacc := similarity-jaccard(gram-tokens($a.nested.title, 3, false), gram-tokens($b.nested.title, 3, false))
-where $jacc >= 0.5f and $a.nested.id < $b.nested.id
-return {"atitle": $a.nested.title, "btitle": $b.nested.title, "jacc": $jacc}
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/ngram-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/ngram-jaccard_01.aql
deleted file mode 100644
index a0f751d..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/ngram-jaccard_01.aql
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index ngram_index on CSX(nested.title) type ngram(3);
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard_01.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where similarity-jaccard(gram-tokens($a.nested.title, 3, false), gram-tokens($b.nested.title, 3, false)) >= 0.5f
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/word-fuzzyeq-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/word-fuzzyeq-jaccard_01.aql
deleted file mode 100644
index e45b02d..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/word-fuzzyeq-jaccard_01.aql
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index keyword_index on DBLP(nested.title) type keyword;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where word-tokens($a.nested.title) ~= word-tokens($b.nested.title) and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/word-jaccard-check-after-btree-access.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/word-jaccard-check-after-btree-access.aql
deleted file mode 100644
index 0fd522d..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/word-jaccard-check-after-btree-access.aql
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, TweetMessages, based on the similarity-jaccard-check function of its text-messages' word tokens.
- * TweetMessages has a keyword index on text-message and btree index on the primary key tweetid, and we expect the join to be
- * transformed into btree and inverted indexed nested-loop joins. We test whether the join condition can be transformed into
- * multiple indexed nested loop joins of various type of indexes.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type TwitterUserType as closed {
- screen-name: string,
- lang: string,
- friends-count: int32,
- statuses-count: int32,
- name: string,
- followers-count: int32
-}
-
-create type TweetMessageNestedType as closed {
- tweetid: int64,
- user: TwitterUserType,
- sender-location: point,
- send-time: datetime,
- referred-topics: {{ string }},
- message-text: string,
- countA: int32,
- countB: int32
-}
-
-create type TweetMessageType as closed {
- nested: TweetMessageNestedType
-}
-
-create dataset TweetMessages(TweetMessageType)
-primary key nested.tweetid;
-
-create index twmSndLocIx on TweetMessages(nested.sender-location) type rtree;
-create index msgCountAIx on TweetMessages(nested.countA) type btree;
-create index msgCountBIx on TweetMessages(nested.countB) type btree;
-create index msgTextIx on TweetMessages(nested.message-text) type keyword;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard-check-after-btree-access.adm";
-
-for $t1 in dataset('TweetMessages')
-for $t2 in dataset('TweetMessages')
-let $sim := similarity-jaccard-check(word-tokens($t1.nested.message-text), word-tokens($t2.nested.message-text), 0.6f)
-where $sim[0] and $t1.nested.tweetid < int64("20") and $t2.nested.tweetid != $t1.nested.tweetid
-return {
- "t1": $t1.nested.tweetid,
- "t2": $t2.nested.tweetid,
- "sim": $sim[1]
-}
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/word-jaccard-check_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/word-jaccard-check_01.aql
deleted file mode 100644
index 55c2ff4..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/word-jaccard-check_01.aql
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard-check function of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index keyword_index on DBLP(nested.title) type keyword;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard-check_01.adm";
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where similarity-jaccard-check(word-tokens($a.nested.title), word-tokens($b.nested.title), 0.5f)[0]
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/word-jaccard-inline.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/word-jaccard-inline.aql
deleted file mode 100644
index 93bcc3e..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/word-jaccard-inline.aql
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, DBLP, based on the similarity-jaccard function of its titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
- * We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create index keyword_index on DBLP(nested.title) type keyword;
-
-write output to asterix_nc1:"rttest/inverted-index-join-noeqjoin_word-jaccard-inline.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-let $jacc := similarity-jaccard(word-tokens($a.nested.title), word-tokens($b.nested.title))
-where $jacc >= 0.5f and $a.nested.id < $b.nested.id
-return {"atitle": $a.nested.title, "btitle": $b.nested.title, "jacc": $jacc}
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/word-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/word-jaccard_01.aql
deleted file mode 100644
index acbe3db..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-index/inverted-index-join/word-jaccard_01.aql
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index keyword_index on DBLP(nested.title) type keyword;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard_01.adm";
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where similarity-jaccard(word-tokens($a.nested.title), word-tokens($b.nested.title)) >= 0.5f
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
-
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-fuzzyeq-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-fuzzyeq-jaccard_01.aql
deleted file mode 100644
index 24dab1c..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-fuzzyeq-jaccard_01.aql
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index ngram_index on DBLP(nested.title: string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where gram-tokens($a.nested.title, 3, false) ~= gram-tokens($b.nested.title, 3, false) and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-fuzzyeq-jaccard_02.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-fuzzyeq-jaccard_02.aql
deleted file mode 100644
index 2f7c560..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-fuzzyeq-jaccard_02.aql
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index ngram_index on CSX(nested.title: string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $b in dataset('DBLP')
-for $a in dataset('CSX')
-where gram-tokens($a.nested.title, 3, false) ~= gram-tokens($b.nested.title, 3, false) and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-fuzzyeq-jaccard_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-fuzzyeq-jaccard_03.aql
deleted file mode 100644
index 5e12569..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-fuzzyeq-jaccard_03.aql
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Self joins dataset DBLP, based on ~= using Jaccard of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create index ngram_index on DBLP(nested.title: string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-where gram-tokens($a.nested.title, 3, false) ~= gram-tokens($b.nested.title, 3, false) and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-fuzzyeq-jaccard_04.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-fuzzyeq-jaccard_04.aql
deleted file mode 100644
index cc2f6cc..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-fuzzyeq-jaccard_04.aql
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index ngram_index_DBLP on DBLP(nested.title: string?) type ngram(3) enforced;
-
-create index ngram_index_CSX on CSX(nested.title: string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where gram-tokens($a.nested.title, 3, false) ~= gram-tokens($b.nested.title, 3, false) and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard-check_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard-check_01.aql
deleted file mode 100644
index da4d914..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard-check_01.aql
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard-check function of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index ngram_index on DBLP(nested.title: string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard-check_01.adm";
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where similarity-jaccard-check(gram-tokens($a.nested.title, 3, false), gram-tokens($b.nested.title, 3, false), 0.5f)[0]
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard-check_02.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard-check_02.aql
deleted file mode 100644
index 5576768..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard-check_02.aql
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard-check function of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index ngram_index on CSX(nested.title: string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard-check_01.adm";
-
-for $b in dataset('DBLP')
-for $a in dataset('CSX')
-where similarity-jaccard-check(gram-tokens($a.nested.title, 3, false), gram-tokens($b.nested.title, 3, false), 0.5f)[0]
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard-check_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard-check_03.aql
deleted file mode 100644
index 32829fb..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard-check_03.aql
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Self joins dataset DBLP, based on the similarity-jaccard-check function of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create index ngram_index on DBLP(nested.title: string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard-check_01.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-where similarity-jaccard-check(gram-tokens($a.nested.title, 3, false), gram-tokens($b.nested.title, 3, false), 0.5f)[0]
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard-check_04.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard-check_04.aql
deleted file mode 100644
index dc4c210..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard-check_04.aql
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard-check function of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index ngram_index_DBLP on DBLP(nested.title: string?) type ngram(3) enforced;
-
-create index ngram_index_CSX on CSX(nested.title: string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard-check_01.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where similarity-jaccard-check(gram-tokens($a.nested.title, 3, false), gram-tokens($b.nested.title, 3, false), 0.5f)[0]
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard-inline.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard-inline.aql
deleted file mode 100644
index bd599b4..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard-inline.aql
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, DBLP, based on the similarity-jaccard function of its titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
- * We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create index ngram_index on DBLP(nested.title: string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join-noeqjoin_ngram-jaccard-inline.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-let $jacc := similarity-jaccard(gram-tokens($a.nested.title, 3, false), gram-tokens($b.nested.title, 3, false))
-where $jacc >= 0.5f and $a.nested.id < $b.nested.id
-return {"atitle": $a.nested.title, "btitle": $b.nested.title, "jacc": $jacc}
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard_01.aql
deleted file mode 100644
index 0106176..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard_01.aql
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index ngram_index on DBLP(nested.title: string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard_01.adm";
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where similarity-jaccard(gram-tokens($a.nested.title, 3, false), gram-tokens($b.nested.title, 3, false)) >= 0.5f
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard_02.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard_02.aql
deleted file mode 100644
index e9f491b..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard_02.aql
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index ngram_index on CSX(nested.title: string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard_01.adm";
-
-for $b in dataset('DBLP')
-for $a in dataset('CSX')
-where similarity-jaccard(gram-tokens($a.nested.title, 3, false), gram-tokens($b.nested.title, 3, false)) >= 0.5f
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard_03.aql
deleted file mode 100644
index e064c03..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard_03.aql
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Self joins dataset DBLP, based on the similarity-jaccard function of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create index ngram_index on DBLP(nested.title: string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard_01.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-where similarity-jaccard(gram-tokens($a.nested.title, 3, false), gram-tokens($b.nested.title, 3, false)) >= 0.5f
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard_04.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard_04.aql
deleted file mode 100644
index b08eabf..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/ngram-jaccard_04.aql
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' 3-gram tokens.
- * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index ngram_index_DBLP on DBLP(nested.title: string?) type ngram(3) enforced;
-
-create index ngram_index_CSX on CSX(nested.title: string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard_01.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where similarity-jaccard(gram-tokens($a.nested.title, 3, false), gram-tokens($b.nested.title, 3, false)) >= 0.5f
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-fuzzyeq-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-fuzzyeq-jaccard_01.aql
deleted file mode 100644
index 287b338..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-fuzzyeq-jaccard_01.aql
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index keyword_index on DBLP(nested.title: string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where word-tokens($a.nested.title) ~= word-tokens($b.nested.title) and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-fuzzyeq-jaccard_02.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-fuzzyeq-jaccard_02.aql
deleted file mode 100644
index e19b4af..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-fuzzyeq-jaccard_02.aql
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index keyword_index on CSX(nested.title: string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $b in dataset('DBLP')
-for $a in dataset('CSX')
-where word-tokens($a.nested.title) ~= word-tokens($b.nested.title) and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-fuzzyeq-jaccard_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-fuzzyeq-jaccard_03.aql
deleted file mode 100644
index 5fd64c9..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-fuzzyeq-jaccard_03.aql
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Self joins dataset DBLP, based on ~= using Jaccard of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create index keyword_index on DBLP(nested.title: string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-where word-tokens($a.nested.title) ~= word-tokens($b.nested.title) and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-fuzzyeq-jaccard_04.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-fuzzyeq-jaccard_04.aql
deleted file mode 100644
index cad00d1..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-fuzzyeq-jaccard_04.aql
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index keyword_index_DBLP on DBLP(nested.title: string?) type keyword enforced;
-
-create index keyword_index_CSX on CSX(nested.title: string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where word-tokens($a.nested.title) ~= word-tokens($b.nested.title) and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-check-after-btree-access.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-check-after-btree-access.aql
deleted file mode 100644
index 9ef8bb2..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-check-after-btree-access.aql
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, TweetMessages, based on the similarity-jaccard-check function of its text-messages' word tokens.
- * TweetMessages has a keyword index on text-message and btree index on the primary key tweetid, and we expect the join to be
- * transformed into btree and inverted indexed nested-loop joins. We test whether the join condition can be transformed into
- * multiple indexed nested loop joins of various type of indexes.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type TwitterUserType as closed {
- screen-name: string,
- lang: string,
- friends-count: int32,
- statuses-count: int32,
- name: string,
- followers-count: int32
-}
-
-create type TweetMessageNestedType as open {
- tweetid: int64,
- user: TwitterUserType,
- sender-location: point,
- send-time: datetime,
- referred-topics: {{ string }},
- countA: int32,
- countB: int32
-}
-
-create type TweetMessageType as closed {
- nested: TweetMessageNestedType
-}
-
-create dataset TweetMessages(TweetMessageType)
-primary key nested.tweetid;
-
-create index twmSndLocIx on TweetMessages(nested.sender-location) type rtree;
-create index msgCountAIx on TweetMessages(nested.countA) type btree;
-create index msgCountBIx on TweetMessages(nested.countB) type btree;
-create index msgTextIx on TweetMessages(nested.message-text: string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard-check-after-btree-access.adm";
-
-for $t1 in dataset('TweetMessages')
-for $t2 in dataset('TweetMessages')
-let $sim := similarity-jaccard-check(word-tokens($t1.nested.message-text), word-tokens($t2.nested.message-text), 0.6f)
-where $sim[0] and $t1.nested.tweetid < int64("20") and $t2.nested.tweetid != $t1.nested.tweetid
-return {
- "t1": $t1.nested.tweetid,
- "t2": $t2.nested.tweetid,
- "sim": $sim[1]
-}
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-check_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-check_01.aql
deleted file mode 100644
index 682a754..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-check_01.aql
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard-check function of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index keyword_index on DBLP(nested.title: string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard-check_01.adm";
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where similarity-jaccard-check(word-tokens($a.nested.title), word-tokens($b.nested.title), 0.5f)[0]
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-check_02.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-check_02.aql
deleted file mode 100644
index 583dda6..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-check_02.aql
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard-check function of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index keyword_index on CSX(nested.title: string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard-check_01.adm";
-
-for $b in dataset('DBLP')
-for $a in dataset('CSX')
-where similarity-jaccard-check(word-tokens($a.nested.title), word-tokens($b.nested.title), 0.5f)[0]
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-check_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-check_03.aql
deleted file mode 100644
index 2cff649..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-check_03.aql
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Selg joins dataset DBLP, based on the similarity-jaccard-check function of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create index keyword_index_DBLP on DBLP(nested.title: string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard-check_01.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-where similarity-jaccard-check(word-tokens($a.nested.title), word-tokens($b.nested.title), 0.5f)[0]
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-check_04.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-check_04.aql
deleted file mode 100644
index fe7b2f9..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-check_04.aql
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard-check function of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index keyword_index on DBLP(nested.title: string?) type keyword enforced;
-
-create index keyword_index on CSX(nested.title: string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard-check_01.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where similarity-jaccard-check(word-tokens($a.nested.title), word-tokens($b.nested.title), 0.5f)[0]
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-inline.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-inline.aql
deleted file mode 100644
index 2ec1846..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard-inline.aql
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, DBLP, based on the similarity-jaccard function of its titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
- * We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create index keyword_index on DBLP(nested.title: string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join-noeqjoin_word-jaccard-inline.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-let $jacc := similarity-jaccard(word-tokens($a.nested.title), word-tokens($b.nested.title))
-where $jacc >= 0.5f and $a.nested.id < $b.nested.id
-return {"atitle": $a.nested.title, "btitle": $b.nested.title, "jacc": $jacc}
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard_01.aql
deleted file mode 100644
index c255829..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard_01.aql
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index keyword_index on DBLP(nested.title: string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard_01.adm";
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where similarity-jaccard(word-tokens($a.nested.title), word-tokens($b.nested.title)) >= 0.5f
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
-
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard_02.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard_02.aql
deleted file mode 100644
index 2d52ae9..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard_02.aql
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index keyword_index on CSX(nested.title: string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard_01.adm";
-
-for $b in dataset('DBLP')
-for $a in dataset('CSX')
-where similarity-jaccard(word-tokens($a.nested.title), word-tokens($b.nested.title)) >= 0.5f
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
-
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard_03.aql
deleted file mode 100644
index 456ad8c..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard_03.aql
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Self joins dataset DBLP, based on the similarity-jaccard function of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create index keyword_index on DBLP(nested.title: string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard_01.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-where similarity-jaccard(word-tokens($a.nested.title), word-tokens($b.nested.title)) >= 0.5f
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
-
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard_04.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard_04.aql
deleted file mode 100644
index 8812d81..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/nested-open-index/inverted-index-join/word-jaccard_04.aql
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' word tokens.
- * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPTypetmp as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXTypetmp as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create type DBLPType as closed {
- nested : DBLPTypetmp
-}
-
-create type CSXType as closed {
- nested : CSXTypetmp
-}
-
-create dataset DBLP(DBLPType) primary key nested.id;
-
-create dataset CSX(CSXType) primary key nested.id;
-
-create index keyword_index on DBLP(nested.title: string?) type keyword enforced;
-
-create index keyword_index on CSX(nested.title: string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard_01.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where similarity-jaccard(word-tokens($a.nested.title), word-tokens($b.nested.title)) >= 0.5f
- and $a.nested.id < $b.nested.id
-return {"arec": $a, "brec": $b }
-
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-fuzzyeq-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-fuzzyeq-jaccard_01.aql
deleted file mode 100644
index 114167e..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-fuzzyeq-jaccard_01.aql
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, open DBLP and closed CSX, based on ~= using Jaccard of their titles' 3-gram tokens.
- * DBLP has a 3-gram enforced open index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index ngram_index on DBLP(title:string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where gram-tokens($a.title, 3, false) ~= gram-tokens($b.title, 3, false) and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-fuzzyeq-jaccard_02.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-fuzzyeq-jaccard_02.aql
deleted file mode 100644
index 237ac57..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-fuzzyeq-jaccard_02.aql
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, closed DBLP and open CSX, based on ~= using Jaccard their titles' 3-gram tokens.
- * CSX has a 3-gram enforced open index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index ngram_index on CSX(title:string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-fuzzyeq-jaccard_02.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where gram-tokens($a.title, 3, false) ~= gram-tokens($b.title, 3, false) and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-fuzzyeq-jaccard_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-fuzzyeq-jaccard_03.aql
deleted file mode 100644
index 6ef70de..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-fuzzyeq-jaccard_03.aql
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, DBLP, based on ~= using Jaccard of its titles' 3-gram tokens.
- * DBLP has a 3-gram enforced open index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create index ngram_index on DBLP(title:string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-fuzzyeq-jaccard_03.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-where gram-tokens($a.title, 3, false) ~= gram-tokens($b.title, 3, false) and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-fuzzyeq-jaccard_04.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-fuzzyeq-jaccard_04.aql
deleted file mode 100644
index 4ddb71f..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-fuzzyeq-jaccard_04.aql
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' 3-gram tokens.
- * DBLP and CSX both have a 3-gram enforced open index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index ngram_index_DBLP on DBLP(title:string?) type ngram(3) enforced;
-
-create index ngram_index_CSX on CSX(title:string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where gram-tokens($a.title, 3, false) ~= gram-tokens($b.title, 3, false) and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard-check_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard-check_01.aql
deleted file mode 100644
index 1277050..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard-check_01.aql
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, open DBLP and closed CSX, based on the similarity-jaccard-check function of their titles' 3-gram tokens.
- * DBLP has a 3-gram enforced open index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index ngram_index on DBLP(title:string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard-check_01.adm";
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where similarity-jaccard-check(gram-tokens($a.title, 3, false), gram-tokens($b.title, 3, false), 0.5f)[0]
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard-check_02.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard-check_02.aql
deleted file mode 100644
index bbaa284..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard-check_02.aql
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, closed DBLP and open CSX, based the similarity-jaccard-check function of their titles' 3-gram tokens.
- * CSX has a 3-gram enforced open index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index ngram_index on CSX(title:string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard-check_02.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where similarity-jaccard-check(gram-tokens($a.title, 3, false), gram-tokens($b.title, 3, false), 0.5f)[0]
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard-check_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard-check_03.aql
deleted file mode 100644
index a0f8683..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard-check_03.aql
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins an open dataset DBLP, based on the similarity-jaccard-check function of its titles' 3-gram tokens.
- * DBLP has a 3-gram enforced open index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create index ngram_index on DBLP(title:string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard-check_03.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-where similarity-jaccard-check(gram-tokens($a.title, 3, false), gram-tokens($b.title, 3, false), 0.5f)[0]
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard-check_04.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard-check_04.aql
deleted file mode 100644
index 26e2504..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard-check_04.aql
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based the similarity-jaccard-check function of their titles' 3-gram tokens.
- * DBLP and CSX both have a 3-gram enforced open index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index ngram_index_DBLP on DBLP(title:string?) type ngram(3) enforced;
-
-create index ngram_index_CSX on CSX(title:string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard-check_02.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where similarity-jaccard-check(gram-tokens($a.title, 3, false), gram-tokens($b.title, 3, false), 0.5f)[0]
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard-check_inline_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard-check_inline_03.aql
deleted file mode 100644
index f061fcf..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard-check_inline_03.aql
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, DBLP, based on the similarity-jaccard-check function of its titles' 3-gram tokens.
- * DBLP has a 3-gram enforced open index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create index ngram_index on DBLP(title:string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard-check_04.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-let $jacc := similarity-jaccard-check(gram-tokens($a.title, 3, false), gram-tokens($b.title, 3, false), 0.5f)
-where $jacc[0] and $a.id < $b.id
-return {"arec": $a, "brec": $b, "jacc": $jacc[1] }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard_01.aql
deleted file mode 100644
index 9f4bd45..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard_01.aql
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' 3-gram tokens.
- * DBLP has a 3-gram enforced open index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index ngram_index on DBLP(title:string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard_01.adm";
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where similarity-jaccard(gram-tokens($a.title, 3, false), gram-tokens($b.title, 3, false)) >= 0.5f
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard_02.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard_02.aql
deleted file mode 100644
index 82b31c3..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard_02.aql
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based the similarity-jaccard function of their titles' 3-gram tokens.
- * CSX has a 3-gram enforced open index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index ngram_index on CSX(title:string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard_02.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where similarity-jaccard(gram-tokens($a.title, 3, false), gram-tokens($b.title, 3, false)) >= 0.5f
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard_03.aql
deleted file mode 100644
index e702d56..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard_03.aql
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, DBLP, based on the similarity-jaccard function of its titles' 3-gram tokens.
- * DBLP has a 3-gram enforced open index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create index ngram_index on DBLP(title:string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard_03.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-where similarity-jaccard(gram-tokens($a.title, 3, false), gram-tokens($b.title, 3, false)) >= 0.5f
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard_04.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard_04.aql
deleted file mode 100644
index e1a9164..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard_04.aql
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based the similarity-jaccard function of their titles' 3-gram tokens.
- * DBLP and CSX both have a 3-gram enforced open index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index ngram_index_DBLP on DBLP(title:string?) type ngram(3) enforced;
-
-create index ngram_index_CSX on CSX(title:string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard_02.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where similarity-jaccard(gram-tokens($a.title, 3, false), gram-tokens($b.title, 3, false)) >= 0.5f
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard_inline_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard_inline_03.aql
deleted file mode 100644
index b634cbf..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/ngram-jaccard_inline_03.aql
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, DBLP, based on the similarity-jaccard function of its titles' 3-gram tokens.
- * DBLP has a 3-gram enforced open index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-set import-private-functions 'true';
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create index ngram_index on DBLP(title:string?) type ngram(3) enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_ngram-jaccard_04.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-let $jacc := similarity-jaccard(gram-tokens($a.title, 3, false), gram-tokens($b.title, 3, false))
-where $jacc >= 0.5f and $a.id < $b.id
-return {"arec": $a, "brec": $b, "jacc": $jacc }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-fuzzyeq-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-fuzzyeq-jaccard_01.aql
deleted file mode 100644
index 3361405..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-fuzzyeq-jaccard_01.aql
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' word tokens.
- * DBLP has an enforced open keyword index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index keyword_index on DBLP(title:string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where word-tokens($a.title) ~= word-tokens($b.title) and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-fuzzyeq-jaccard_02.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-fuzzyeq-jaccard_02.aql
deleted file mode 100644
index 3990354..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-fuzzyeq-jaccard_02.aql
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' word tokens.
- * CSX has an enforced open keyword index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index keyword_index on CSX(title:string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-fuzzyeq-jaccard_02.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where word-tokens($a.title) ~= word-tokens($b.title) and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-fuzzyeq-jaccard_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-fuzzyeq-jaccard_03.aql
deleted file mode 100644
index 9cccbf0..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-fuzzyeq-jaccard_03.aql
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, DBLP, based on ~= using Jaccard of its titles' word tokens.
- * DBLP has an enforced open keyword index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create index keyword_index on DBLP(title:string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-fuzzyeq-jaccard_03.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-where word-tokens($a.title) ~= word-tokens($b.title) and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-fuzzyeq-jaccard_04.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-fuzzyeq-jaccard_04.aql
deleted file mode 100644
index 0a0eb66..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-fuzzyeq-jaccard_04.aql
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' word tokens.
- * DBLP and CSX both have an enforced open keyword index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index keyword_index_DBLP on DBLP(title:string?) type keyword enforced;
-
-create index keyword_index_CSX on CSX(title:string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-fuzzyeq-jaccard_01.adm";
-
-set simfunction 'jaccard';
-set simthreshold '0.5f';
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where word-tokens($a.title) ~= word-tokens($b.title) and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check-after-btree-access.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check-after-btree-access.aql
deleted file mode 100644
index d090e0b..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check-after-btree-access.aql
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, TweetMessages, based on the similarity-jaccard-check function of its text-messages' word tokens.
- * TweetMessages has a keyword index on text-message and btree index on the primary key tweetid, and we expect the join to be
- * transformed into btree and inverted indexed nested-loop joins. We test whether the join condition can be transformed into
- * multiple indexed nested loop joins of various type of indexes.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type TwitterUserType as closed {
- screen-name: string,
- lang: string,
- friends-count: int32,
- statuses-count: int32,
- name: string,
- followers-count: int32
-}
-
-create type TweetMessageType as open {
- tweetid: int64,
- user: TwitterUserType,
- sender-location: point,
- send-time: datetime,
- referred-topics: {{ string }},
- countA: int32,
- countB: int32
-}
-
-create dataset TweetMessages(TweetMessageType)
-primary key tweetid;
-
-create index twmSndLocIx on TweetMessages(sender-location) type rtree;
-create index msgCountAIx on TweetMessages(countA) type btree;
-create index msgCountBIx on TweetMessages(countB) type btree;
-create index msgTextIx on TweetMessages(message-text: string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard-check-after-btree-access.adm";
-
-for $t1 in dataset('TweetMessages')
-for $t2 in dataset('TweetMessages')
-let $sim := similarity-jaccard-check(word-tokens($t1.message-text), word-tokens($t2.message-text), 0.6f)
-where $sim[0] and $t1.tweetid < int64("20") and $t2.tweetid != $t1.tweetid
-return {
- "t1": $t1.tweetid,
- "t2": $t2.tweetid,
- "sim": $sim[1]
-}
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check_01.aql
deleted file mode 100644
index e784002..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check_01.aql
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard-check function of their titles' word tokens.
- * DBLP has an enforced open keyword index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index keyword_index on DBLP(title:string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard-check_01.adm";
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where similarity-jaccard-check(word-tokens($a.title), word-tokens($b.title), 0.5f)[0]
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check_02.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check_02.aql
deleted file mode 100644
index 27a170d..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check_02.aql
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard-check function of their titles' word tokens.
- * CSX has an enforced open keyword index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index keyword_index on CSX(title:string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard-check_02.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where similarity-jaccard-check(word-tokens($a.title), word-tokens($b.title), 0.5f)[0]
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check_03.aql
deleted file mode 100644
index 7fca777..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check_03.aql
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, DBLP, based on the similarity-jaccard-check function of its titles' word tokens.
- * DBLP has an enforced open keyword index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create index keyword_index on DBLP(title:string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard-check_03.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-where similarity-jaccard-check(word-tokens($a.title), word-tokens($b.title), 0.5f)[0]
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check_04.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check_04.aql
deleted file mode 100644
index 294732f..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check_04.aql
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard-check function of their titles' word tokens.
- * DBLP and CSX both have an enforced open keyword index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index keyword_index_DBLP on DBLP(title:string?) type keyword enforced;
-
-create index keyword_index_CSX on CSX(title:string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard-check_01.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where similarity-jaccard-check(word-tokens($a.title), word-tokens($b.title), 0.5f)[0]
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check_inline_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check_inline_03.aql
deleted file mode 100644
index e2f32f3..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard-check_inline_03.aql
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, DBLP, based on the similarity-jaccard-check function of its titles' word tokens.
- * DBLP has an enforced open keyword index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create index keyword_index on DBLP(title:string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard-check_04.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-let $jacc := similarity-jaccard-check(word-tokens($a.title), word-tokens($b.title), 0.5f)
-where $jacc[0] and $a.id < $b.id
-return {"arec": $a, "brec": $b, "jacc": $jacc[1] }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard_01.aql
deleted file mode 100644
index 7f2ac1c..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard_01.aql
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' word tokens.
- * DBLP has an enforced open keyword index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as closed {
- id: int32,
- csxid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index keyword_index on DBLP(title:string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard_01.adm";
-
-for $b in dataset('CSX')
-for $a in dataset('DBLP')
-where similarity-jaccard(word-tokens($a.title), word-tokens($b.title)) >= 0.5f
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
-
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard_02.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard_02.aql
deleted file mode 100644
index a143cff..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard_02.aql
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' word tokens.
- * CSX has an enforced open keyword index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index keyword_index on CSX(title:string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard_02.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where similarity-jaccard(word-tokens($a.title), word-tokens($b.title)) >= 0.5f
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
-
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard_03.aql
deleted file mode 100644
index 744505b..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard_03.aql
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, DBLP, based on the similarity-jaccard function of its titles' word tokens.
- * DBLP has an enforced open keyword index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create index keyword_index on DBLP(title:string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard_03.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-where similarity-jaccard(word-tokens($a.title), word-tokens($b.title)) >= 0.5f
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard_04.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard_04.aql
deleted file mode 100644
index f49113a..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard_04.aql
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' word tokens.
- * DBLP and CSX both have an enforced open keyword index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create type CSXType as open {
- id: int32,
- csxid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create dataset CSX(CSXType) primary key id;
-
-create index keyword_index_DBLP on DBLP(title:string?) type keyword enforced;
-
-create index keyword_index_CSX on CSX(title:string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard_01.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('CSX')
-where similarity-jaccard(word-tokens($a.title), word-tokens($b.title)) >= 0.5f
- and $a.id < $b.id
-return {"arec": $a, "brec": $b }
-
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard_inline_03.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard_inline_03.aql
deleted file mode 100644
index 81f0a77..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/open-index-enforced/inverted-index-join/word-jaccard_inline_03.aql
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Fuzzy self joins a dataset, DBLP, based on the similarity-jaccard function of its titles' word tokens.
- * DBLP has an enforced open keyword index on title?, and we expect the join to be transformed into an indexed nested-loop join.
- * We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
- * Success : Yes
- */
-
-drop dataverse test if exists;
-create dataverse test;
-use dataverse test;
-
-create type DBLPType as open {
- id: int32,
- dblpid: string,
- authors: string,
- misc: string
-}
-
-create dataset DBLP(DBLPType) primary key id;
-
-create index keyword_index on DBLP(title:string?) type keyword enforced;
-
-write output to asterix_nc1:"rttest/inverted-index-join_word-jaccard_04.adm";
-
-for $a in dataset('DBLP')
-for $b in dataset('DBLP')
-let $jacc := similarity-jaccard(word-tokens($a.title), word-tokens($b.title))
-where $jacc >= 0.5f and $a.id < $b.id
-return {"arec": $a, "brec": $b, "jacc": $jacc }
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/similarity/jaccard-similarity-join-dual-order.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/similarity/jaccard-similarity-join-dual-order.aql
deleted file mode 100644
index ca88a9e..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/similarity/jaccard-similarity-join-dual-order.aql
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description : Tests the plan of runtimes/fuzzyjoin/basic_1_3_1/6
- * Success : Yes
- */
-
-drop dataverse fuzzyjoin_basic if exists;
-
-create dataverse fuzzyjoin_basic;
-
-use dataverse fuzzyjoin_basic;
-
-create type BasicType as closed {
- id: uuid,
- authors: string
-}
-
-create dataset left(BasicType) primary key id autogenerated;
-create dataset right(BasicType) primary key id autogenerated;
-
-write output to asterix_nc1:"rttest/jaccard-similarity-join-right-ahead.adm";
-
-set import-private-functions 'true'
-
-for $right in dataset('right')
-let $idRight := $right.id
-let $tokensUnrankedRight := word-tokens($right.authors)
-let $lenRight := len($tokensUnrankedRight)
-let $tokensRight :=
- for $tokenUnranked in $tokensUnrankedRight
- for $tokenRanked at $i in
- //
- // -- - Stage 1 - --
- //
- for $r in
- for $orderRight in dataset('right')
- let $rightId := $orderRight.id
- for $orderTokenRight in word-tokens($orderRight.authors)
- /*+ hash */ group by $tokenRightGrouped := $orderTokenRight with $rightId
- return {"rt": $tokenRightGrouped, "rc": count($rightId)}
- for $l in
- for $orderLeft in dataset('left')
- let $leftId := $orderLeft.id
- for $orderTokenLeft in word-tokens($orderLeft.authors)
- /*+ hash */ group by $tokenLeftGrouped := $orderTokenLeft with $leftId
- return {"lt": $tokenLeftGrouped, "lc": count($leftId)}
- where $r.rt = $l.lt
- /*+ inmem 1 302 */ order by $r.rc * $l.lc
- return $r.rt
-
- where $tokenUnranked = /*+ hash-bcast */ $tokenRanked
- order by $i
- return $i
-for $prefixTokenRight in subset-collection($tokensRight, 0, prefix-len-jaccard(len($tokensRight), .8f))
-
-for $left in dataset('left')
-let $idLeft := $left.id
-let $tokensUnrankedLeft := word-tokens($left.authors)
-let $lenLeft := len($tokensUnrankedLeft)
-let $tokensLeft :=
- for $tokenUnranked in $tokensUnrankedLeft
- for $tokenRanked at $i in
- //
- // -- - Stage 1 - --
- //
- for $r in
- for $orderRight in dataset('right')
- let $rightId := $orderRight.id
- for $orderTokenRight in word-tokens($orderRight.authors)
- /*+ hash */ group by $tokenRightGrouped := $orderTokenRight with $rightId
- return {"rt": $tokenRightGrouped, "rc": count($rightId)}
- for $l in
- for $orderLeft in dataset('left')
- let $leftId := $orderLeft.id
- for $orderTokenLeft in word-tokens($orderLeft.authors)
- /*+ hash */ group by $tokenLeftGrouped := $orderTokenLeft with $leftId
- return {"lt": $tokenLeftGrouped, "lc": count($leftId)}
- where $r.rt = $l.lt
- /*+ inmem 1 302 */ order by $r.rc * $l.lc
- return $r.rt
-
- where $tokenUnranked = /*+ hash-bcast */ $tokenRanked
- order by $i
- return $i
-let $actualPrefixLen := prefix-len-jaccard(len($tokensUnrankedLeft), .8f) - len($tokensUnrankedLeft) + len($tokensLeft)
-for $prefixTokenLeft in subset-collection($tokensLeft, 0, $actualPrefixLen)
-
-where $prefixTokenRight = $prefixTokenLeft
-let $sim := similarity-jaccard-prefix($lenRight, $tokensRight, $lenLeft, $tokensLeft, $prefixTokenLeft, .8f)
-where $sim >= .8f
-/*+ hash*/ group by $idRight := $idRight, $idLeft := $idLeft with $sim
-return {'idDBLP': $idRight, 'idCSX': $idLeft, "sim": $sim[0]}
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/unnest_list_in_subplan.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/unnest_list_in_subplan.aql
deleted file mode 100644
index 2c5bf45..0000000
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/unnest_list_in_subplan.aql
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-drop dataverse fuzzyjoin if exists;
-create dataverse fuzzyjoin;
-use dataverse fuzzyjoin;
-
-create type DBLPType as closed {
- id: int32,
- dblpid: string,
- title: string,
- authors: string,
- misc: string
-}
-
-create type TOKENSRANKEDADMType as closed {
- token: int32,
- rank: int32
-}
-
-create nodegroup group1 if not exists on asterix_nc1, asterix_nc2;
-
-create dataset DBLP(DBLPType) primary key id on group1;
-create dataset TOKENSRANKEDADM(TOKENSRANKEDADMType) primary key rank on group1;
-
-write output to asterix_nc1:'rttest/unnest_list_in_subplan.adm';
-
- //
- // -- - Stage 2 - --
- //
- for $paperDBLP in dataset('DBLP')
- let $idDBLP := $paperDBLP.id
- let $tokensUnrankedDBLP := counthashed-word-tokens($paperDBLP.title)
- let $tokensDBLP :=
- for $tokenUnranked in $tokensUnrankedDBLP
- for $tokenRanked in dataset('TOKENSRANKEDADM')
- where $tokenUnranked = $tokenRanked.token
- order by $tokenRanked.rank
- return $tokenRanked.rank
- return {'id': $idDBLP, 'tokens': $tokensDBLP}
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join-noeqjoin/ngram-jaccard-inline.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join-noeqjoin/ngram-jaccard-inline.plan
index 1a36d3c..8860a1e 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join-noeqjoin/ngram-jaccard-inline.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join-noeqjoin/ngram-jaccard-inline.plan
@@ -78,12 +78,12 @@
-- SORT_MERGE_EXCHANGE [$$130(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$130(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$137] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$136] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$137] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$136] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -133,12 +133,12 @@
-- SORT_MERGE_EXCHANGE [$$130(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$130(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$137] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$136] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$137] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$136] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join-noeqjoin/word-jaccard-inline.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join-noeqjoin/word-jaccard-inline.plan
index 1a36d3c..8860a1e 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join-noeqjoin/word-jaccard-inline.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join-noeqjoin/word-jaccard-inline.plan
@@ -78,12 +78,12 @@
-- SORT_MERGE_EXCHANGE [$$130(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$130(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$137] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$136] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$137] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$136] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -133,12 +133,12 @@
-- SORT_MERGE_EXCHANGE [$$130(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$130(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$137] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$136] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$137] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$136] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/issue741.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/issue741.plan
index 619b74e..d4ebaa9 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/issue741.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/issue741.plan
@@ -84,12 +84,12 @@
-- SORT_MERGE_EXCHANGE [$$140(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$140(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$147] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$146] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$147] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$146] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -143,12 +143,12 @@
-- SORT_MERGE_EXCHANGE [$$140(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$140(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$147] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$146] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$147] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$146] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.plan
index 774794e..56d899c 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.plan
@@ -88,12 +88,12 @@
-- SORT_MERGE_EXCHANGE [$$143(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$143(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$150] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$149] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$150] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$149] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -149,12 +149,12 @@
-- SORT_MERGE_EXCHANGE [$$143(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$143(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$150] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$149] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$150] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$149] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01_ps.plan
index 336a4d4..c358e89 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01_ps.plan
@@ -94,12 +94,12 @@
-- SORT_MERGE_EXCHANGE [$$143(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$143(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$150] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$149] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$150] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$149] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -155,12 +155,12 @@
-- SORT_MERGE_EXCHANGE [$$143(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$143(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$150] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$149] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$150] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$149] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -269,12 +269,12 @@
-- SORT_MERGE_EXCHANGE [$$143(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$143(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$150] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$149] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$150] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$149] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -330,12 +330,12 @@
-- SORT_MERGE_EXCHANGE [$$143(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$143(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$150] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$149] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$150] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$149] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-fuzzyeq-jaccard_02.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-fuzzyeq-jaccard_02.plan
index cbbce46..7cc36bb 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-fuzzyeq-jaccard_02.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-fuzzyeq-jaccard_02.plan
@@ -66,12 +66,12 @@
-- SORT_MERGE_EXCHANGE [$$117(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$117(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$123] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$123] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -121,12 +121,12 @@
-- SORT_MERGE_EXCHANGE [$$117(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$117(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$123] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$123] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-fuzzyeq-jaccard_03.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-fuzzyeq-jaccard_03.plan
index 79e824e..ddd6bcb 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-fuzzyeq-jaccard_03.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-fuzzyeq-jaccard_03.plan
@@ -65,12 +65,12 @@
-- SORT_MERGE_EXCHANGE [$$117(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$117(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$123] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$123] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -120,12 +120,12 @@
-- SORT_MERGE_EXCHANGE [$$117(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$117(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$123] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$123] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard-check_02.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard-check_02.plan
index 1daa1cf..69c0be7 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard-check_02.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard-check_02.plan
@@ -66,12 +66,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -121,12 +121,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard-check_03.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard-check_03.plan
index 01e070e..c2938f0 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard-check_03.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard-check_03.plan
@@ -65,12 +65,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -120,12 +120,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard-check_04.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard-check_04.plan
index 22ad072..0039721 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard-check_04.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard-check_04.plan
@@ -67,12 +67,12 @@
-- SORT_MERGE_EXCHANGE [$$130(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$130(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$137] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$136] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$137] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$136] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -122,12 +122,12 @@
-- SORT_MERGE_EXCHANGE [$$130(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$130(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$137] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$136] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$137] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$136] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard_02.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard_02.plan
index 1daa1cf..69c0be7 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard_02.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard_02.plan
@@ -66,12 +66,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -121,12 +121,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard_03.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard_03.plan
index 01e070e..c2938f0 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard_03.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard_03.plan
@@ -65,12 +65,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -120,12 +120,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard_04.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard_04.plan
index d3acd29..fb60820 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard_04.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/ngram-jaccard_04.plan
@@ -67,12 +67,12 @@
-- SORT_MERGE_EXCHANGE [$$130(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$130(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$137] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$136] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$137] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$136] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -122,12 +122,12 @@
-- SORT_MERGE_EXCHANGE [$$130(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$130(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$137] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$136] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$137] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$136] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-fuzzyeq-jaccard_02.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-fuzzyeq-jaccard_02.plan
index cbbce46..7cc36bb 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-fuzzyeq-jaccard_02.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-fuzzyeq-jaccard_02.plan
@@ -66,12 +66,12 @@
-- SORT_MERGE_EXCHANGE [$$117(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$117(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$123] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$123] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -121,12 +121,12 @@
-- SORT_MERGE_EXCHANGE [$$117(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$117(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$123] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$123] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-fuzzyeq-jaccard_03.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-fuzzyeq-jaccard_03.plan
index 79e824e..ddd6bcb 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-fuzzyeq-jaccard_03.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-fuzzyeq-jaccard_03.plan
@@ -65,12 +65,12 @@
-- SORT_MERGE_EXCHANGE [$$117(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$117(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$123] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$123] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -120,12 +120,12 @@
-- SORT_MERGE_EXCHANGE [$$117(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$117(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$123] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$123] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check-after-btree-access.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check-after-btree-access.plan
index 80d465e..70f44c5 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check-after-btree-access.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check-after-btree-access.plan
@@ -76,12 +76,12 @@
-- SORT_MERGE_EXCHANGE [$$135(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$135(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$142] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$141] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$142] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$141] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -134,12 +134,12 @@
-- SORT_MERGE_EXCHANGE [$$135(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$135(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$142] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$141] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$142] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$141] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check_02.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check_02.plan
index 1daa1cf..69c0be7 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check_02.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check_02.plan
@@ -66,12 +66,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -121,12 +121,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check_03.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check_03.plan
index 01e070e..c2938f0 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check_03.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check_03.plan
@@ -65,12 +65,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -120,12 +120,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check_04.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check_04.plan
index 22ad072..0039721 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check_04.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard-check_04.plan
@@ -67,12 +67,12 @@
-- SORT_MERGE_EXCHANGE [$$130(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$130(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$137] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$136] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$137] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$136] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -122,12 +122,12 @@
-- SORT_MERGE_EXCHANGE [$$130(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$130(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$137] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$136] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$137] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$136] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard_02.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard_02.plan
index 1daa1cf..69c0be7 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard_02.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard_02.plan
@@ -66,12 +66,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -121,12 +121,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard_03.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard_03.plan
index 01e070e..c2938f0 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard_03.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard_03.plan
@@ -65,12 +65,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -120,12 +120,12 @@
-- SORT_MERGE_EXCHANGE [$$118(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$118(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$125] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$124] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$125] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$124] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard_04.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard_04.plan
index d3acd29..fb60820 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard_04.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/inverted-index-join/word-jaccard_04.plan
@@ -67,12 +67,12 @@
-- SORT_MERGE_EXCHANGE [$$130(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$130(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$137] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$136] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$137] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$136] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -122,12 +122,12 @@
-- SORT_MERGE_EXCHANGE [$$130(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$130(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$137] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$136] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$137] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$136] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/nested-index/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01_ps.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/nested-index/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01_ps.plan
index dc19740..8a2df0d 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/nested-index/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01_ps.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/nested-index/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01_ps.plan
@@ -98,12 +98,12 @@
-- SORT_MERGE_EXCHANGE [$$170(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$170(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$177] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$176] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$177] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$176] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -158,12 +158,12 @@
-- SORT_MERGE_EXCHANGE [$$170(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$170(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$177] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$176] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$177] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$176] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -275,12 +275,12 @@
-- SORT_MERGE_EXCHANGE [$$170(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$170(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$177] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$176] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$177] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$176] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
@@ -335,12 +335,12 @@
-- SORT_MERGE_EXCHANGE [$$170(ASC), $$tokenGroupped(ASC) ] |PARTITIONED|
-- STABLE_SORT [$$170(ASC), $$tokenGroupped(ASC)] |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- EXTERNAL_GROUP_BY[$$177] |PARTITIONED|
+ -- EXTERNAL_GROUP_BY[$$176] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
-- NESTED_TUPLE_SOURCE |LOCAL|
}
- -- HASH_PARTITION_EXCHANGE [$$177] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$176] |PARTITIONED|
-- EXTERNAL_GROUP_BY[$$token] |PARTITIONED|
{
-- AGGREGATE |LOCAL|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.1.ddl.sqlpp
new file mode 100644
index 0000000..ab6b723
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.1.ddl.sqlpp
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test1 if exists;
+create dataverse test1;
+use test1;
+
+create dataset a(
+ c_a1 bigint not unknown,
+ c_a2 bigint,
+ c_a3 string
+) primary key c_a1;
+
+create index ia2 on a(c_a2);
+
+create index ia3 on a(c_a3);
+
+create dataset b(
+ c_b1 bigint not unknown,
+ c_b2 bigint,
+ c_b3 string
+) primary key c_b1;
+
+create index ib2 on b(c_b2);
+
+create index ib3 on b(c_b3);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.2.ddl.sqlpp
new file mode 100644
index 0000000..621d057
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.2.ddl.sqlpp
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test2 if exists;
+create dataverse test2;
+use test2;
+
+create dataset a(
+ c_a1 bigint not unknown,
+ c_a2 bigint,
+ c_a3 string
+) primary key c_a1;
+
+create index ia2 on a(c_a2);
+
+create index ia3 on a(c_a3);
+
+create dataset b(
+ c_b1 bigint not unknown,
+ c_b2 bigint,
+ c_b3 string
+) primary key c_b1;
+
+create index ib2 on b(c_b2);
+
+create index ib3 on b(c_b3);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.query.sqlpp
new file mode 100644
index 0000000..e48a1e1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.query.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+select ds.DataverseName, ds.DatasetName, dt.Derived.IsAnonymous, indexes
+from Metadata.`Dataset` as ds left join Metadata.`Datatype` dt
+on ds.DataverseName = dt.DataverseName and ds.DatatypeName = dt.DatatypeName
+let indexes = (
+ select idx.IndexName
+ from Metadata.`Index` as idx
+ where idx.DataverseName = ds.DataverseName and idx.DatasetName = ds.DatasetName
+ order by idx.IndexName
+)
+where ds.DataverseName like "test%"
+order by ds.DataverseName, ds.DatasetName;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.adm
new file mode 100644
index 0000000..7065d25
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2886/query-ASTERIXDB-2886.3.adm
@@ -0,0 +1,4 @@
+{ "DataverseName": "test1", "DatasetName": "a", "IsAnonymous": true, "indexes": [ { "IndexName": "a" }, { "IndexName": "ia2" }, { "IndexName": "ia3" } ] }
+{ "DataverseName": "test1", "DatasetName": "b", "IsAnonymous": true, "indexes": [ { "IndexName": "b" }, { "IndexName": "ib2" }, { "IndexName": "ib3" } ] }
+{ "DataverseName": "test2", "DatasetName": "a", "IsAnonymous": true, "indexes": [ { "IndexName": "a" }, { "IndexName": "ia2" }, { "IndexName": "ia3" } ] }
+{ "DataverseName": "test2", "DatasetName": "b", "IsAnonymous": true, "indexes": [ { "IndexName": "b" }, { "IndexName": "ib2" }, { "IndexName": "ib3" } ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 11c9634..4c034ad 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -6399,6 +6399,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-2886">
+ <output-dir compare="Text">query-ASTERIXDB-2886</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
<compilation-unit name="unsupported_parameter">
<output-dir compare="Text">none</output-dir>
<expected-error>Query parameter compiler.joinmem is not supported</expected-error>
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java
index 5386193..3760be5 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java
@@ -36,7 +36,6 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
@@ -69,26 +68,33 @@ import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule {
private final VariableSubstitutionVisitor substVisitor = new VariableSubstitutionVisitor();
- private final Map<LogicalVariable, List<LogicalVariable>> equivalentVarsMap =
- new HashMap<LogicalVariable, List<LogicalVariable>>();
+
+ private final Map<LogicalVariable, List<LogicalVariable>> equivalentVarsMap = new HashMap<>();
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
if (context.checkIfInDontApplySet(this, opRef.getValue())) {
return false;
}
- boolean modified = removeRedundantVariables(opRef, context);
- if (modified) {
- context.computeAndSetTypeEnvironmentForOperator(opRef.getValue());
- }
- return modified;
+ clear();
+ return removeRedundantVariables(opRef, true, context);
+ }
+
+ private void clear() {
+ equivalentVarsMap.clear();
}
private void updateEquivalenceClassMap(LogicalVariable lhs, LogicalVariable rhs) {
List<LogicalVariable> equivalentVars = equivalentVarsMap.get(rhs);
if (equivalentVars == null) {
- equivalentVars = new ArrayList<LogicalVariable>();
+ equivalentVars = new ArrayList<>();
// The first element in the list is the bottom-most representative which will replace all equivalent vars.
equivalentVars.add(rhs);
equivalentVars.add(lhs);
@@ -97,12 +103,32 @@ public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule {
equivalentVarsMap.put(lhs, equivalentVars);
}
- private boolean removeRedundantVariables(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
- throws AlgebricksException {
+ private LogicalVariable findEquivalentRepresentativeVar(LogicalVariable var) {
+ List<LogicalVariable> equivalentVars = equivalentVarsMap.get(var);
+ if (equivalentVars == null) {
+ return null;
+ }
+ LogicalVariable representativeVar = equivalentVars.get(0);
+ return var.equals(representativeVar) ? null : representativeVar;
+ }
+
+ private boolean removeRedundantVariables(Mutable<ILogicalOperator> opRef, boolean first,
+ IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (!first) {
+ context.addToDontApplySet(this, op);
+ }
+
LogicalOperatorTag opTag = op.getOperatorTag();
boolean modified = false;
+ // Recurse into children.
+ for (Mutable<ILogicalOperator> inputOpRef : op.getInputs()) {
+ if (removeRedundantVariables(inputOpRef, false, context)) {
+ modified = true;
+ }
+ }
+
// Update equivalence class map.
if (opTag == LogicalOperatorTag.ASSIGN) {
AssignOperator assignOp = (AssignOperator) op;
@@ -142,7 +168,7 @@ public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule {
AbstractOperatorWithNestedPlans opWithNestedPlan = (AbstractOperatorWithNestedPlans) op;
for (ILogicalPlan nestedPlan : opWithNestedPlan.getNestedPlans()) {
for (Mutable<ILogicalOperator> rootRef : nestedPlan.getRoots()) {
- if (removeRedundantVariables(rootRef, context)) {
+ if (removeRedundantVariables(rootRef, false, context)) {
modified = true;
}
}
@@ -158,14 +184,8 @@ public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule {
if (modified) {
context.computeAndSetTypeEnvironmentForOperator(op);
- context.addToDontApplySet(this, op);
}
- // Clears the equivalent variable map if the current operator is the root operator
- // in the query plan.
- if (opTag == LogicalOperatorTag.DISTRIBUTE_RESULT || opTag == LogicalOperatorTag.SINK) {
- equivalentVarsMap.clear();
- }
return modified;
}
@@ -227,38 +247,34 @@ public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule {
* We cannot use the VariableSubstitutionVisitor here because the project ops
* maintain their variables as a list and not as expressions.
*/
- private boolean replaceProjectVars(ProjectOperator op) throws AlgebricksException {
+ private boolean replaceProjectVars(ProjectOperator op) {
List<LogicalVariable> vars = op.getVariables();
int size = vars.size();
boolean modified = false;
for (int i = 0; i < size; i++) {
LogicalVariable var = vars.get(i);
- List<LogicalVariable> equivalentVars = equivalentVarsMap.get(var);
- if (equivalentVars == null) {
- continue;
- }
- // Replace with equivalence class representative.
- LogicalVariable representative = equivalentVars.get(0);
- if (representative != var) {
- vars.set(i, equivalentVars.get(0));
+ LogicalVariable representativeVar = findEquivalentRepresentativeVar(var);
+ if (representativeVar != null) {
+ // Replace with equivalence class representative.
+ vars.set(i, representativeVar);
modified = true;
}
}
return modified;
}
- private boolean replaceUnionAllVars(UnionAllOperator op) throws AlgebricksException {
+ private boolean replaceUnionAllVars(UnionAllOperator op) {
boolean modified = false;
for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping : op.getVariableMappings()) {
- List<LogicalVariable> firstEquivalentVars = equivalentVarsMap.get(varMapping.first);
- List<LogicalVariable> secondEquivalentVars = equivalentVarsMap.get(varMapping.second);
// Replace variables with their representative.
- if (firstEquivalentVars != null) {
- varMapping.first = firstEquivalentVars.get(0);
+ LogicalVariable firstRepresentativeVar = findEquivalentRepresentativeVar(varMapping.first);
+ if (firstRepresentativeVar != null) {
+ varMapping.first = firstRepresentativeVar;
modified = true;
}
- if (secondEquivalentVars != null) {
- varMapping.second = secondEquivalentVars.get(0);
+ LogicalVariable secondRepresentativeVar = findEquivalentRepresentativeVar(varMapping.second);
+ if (secondRepresentativeVar != null) {
+ varMapping.second = secondRepresentativeVar;
modified = true;
}
}
@@ -269,17 +285,13 @@ public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule {
@Override
public boolean transform(Mutable<ILogicalExpression> exprRef) {
ILogicalExpression e = exprRef.getValue();
- switch (((AbstractLogicalExpression) e).getExpressionTag()) {
+ switch (e.getExpressionTag()) {
case VARIABLE: {
// Replace variable references with their equivalent representative in the equivalence class map.
VariableReferenceExpression varRefExpr = (VariableReferenceExpression) e;
LogicalVariable var = varRefExpr.getVariableReference();
- List<LogicalVariable> equivalentVars = equivalentVarsMap.get(var);
- if (equivalentVars == null) {
- return false;
- }
- LogicalVariable representative = equivalentVars.get(0);
- if (representative != var) {
+ LogicalVariable representative = findEquivalentRepresentativeVar(var);
+ if (representative != null) {
varRefExpr.setVariable(representative);
return true;
}
[asterixdb] 16/23: [NO ISSUE][HYR] Expose
getParams/getSourceLocation on IFormattedException, Warning
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 02a14247c0b79dddd1085fd769f1032d6eca3d5a
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Fri Jun 25 15:06:14 2021 -0400
[NO ISSUE][HYR] Expose getParams/getSourceLocation on IFormattedException,Warning
Change-Id: Id6c6807bfe727e8164ac69dd8e8a18fcf1840cce
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12064
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
.../apache/hyracks/api/exceptions/IFormattedException.java | 11 +++++++++++
.../java/org/apache/hyracks/api/exceptions/Warning.java | 13 ++++++++++---
2 files changed, 21 insertions(+), 3 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/IFormattedException.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/IFormattedException.java
index fd40e3f..cdd17bd 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/IFormattedException.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/IFormattedException.java
@@ -18,6 +18,7 @@
*/
package org.apache.hyracks.api.exceptions;
+import java.io.Serializable;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Stream;
@@ -53,6 +54,16 @@ public interface IFormattedException {
Optional<IError> getError();
/**
+ * @return the source location
+ */
+ SourceLocation getSourceLocation();
+
+ /**
+ * @return the parameters to use when formatting
+ */
+ Serializable[] getParams();
+
+ /**
* Indicates whether this exception matches the supplied error code
*/
default boolean matches(IError candidate) {
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java
index d82f17d..b38c7ac 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/Warning.java
@@ -33,12 +33,14 @@ public class Warning implements Serializable {
private final SourceLocation srcLocation;
private final int code;
private final String message;
+ private final Serializable[] params;
- private Warning(String component, SourceLocation srcLocation, int code, String message) {
+ private Warning(String component, SourceLocation srcLocation, int code, String message, Serializable... params) {
this.component = component;
this.srcLocation = srcLocation;
this.code = code;
this.message = message;
+ this.params = params;
}
/**
@@ -59,7 +61,7 @@ public class Warning implements Serializable {
public static Warning of(SourceLocation srcLocation, IError code, Serializable... params) {
return new Warning(code.component(), srcLocation, code.intValue(), ErrorMessageUtil
- .formatMessage(code.component(), code.intValue(), code.errorMessage(), srcLocation, params));
+ .formatMessage(code.component(), code.intValue(), code.errorMessage(), srcLocation, params), params);
}
public String getComponent() {
@@ -106,7 +108,8 @@ public class Warning implements Serializable {
String comp = input.readUTF();
int code = input.readInt();
String msg = input.readUTF();
- return new Warning(comp, SourceLocation.create(input), code, msg);
+ SourceLocation sourceLocation = SourceLocation.create(input);
+ return new Warning(comp, sourceLocation, code, msg);
}
@Override
@@ -114,4 +117,8 @@ public class Warning implements Serializable {
return "Warning{" + "component='" + component + '\'' + ", srcLocation=" + srcLocation + ", code=" + code
+ ", message='" + message + '\'' + '}';
}
+
+ public Serializable[] getParams() {
+ return params;
+ }
}
[asterixdb] 02/23: [ASTERIXDB-2797][COMP] Fix casting elements of a
list constructor
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 4b66ee093cd45880db52913d2ac16bd07c7a895b
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Thu Jun 3 17:28:29 2021 +0300
[ASTERIXDB-2797][COMP] Fix casting elements of a list constructor
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Currently, variable elements in a list constructor are always wrapped
in cast() which would cast the variable to open type (when the variable
type is a complex type, e.g. record). The required target type should
be taken into account to decide whether to cast or not.
Change-Id: If18467115e7e497af1b3bc1b76d8ff64adf14c4a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11743
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
---
.../rules/IntroduceEnforcedListTypeRule.java | 1 +
.../rules/typecast/StaticTypeCastUtil.java | 31 +++++++++++++---------
.../list/var-in-list/var-in-list.01.ddl.sqlpp | 30 +++++++++++++++++++++
.../list/var-in-list/var-in-list.02.update.sqlpp | 23 ++++++++++++++++
.../list/var-in-list/var-in-list.03.query.sqlpp | 22 +++++++++++++++
.../list/var-in-list/var-in-list.04.query.sqlpp | 22 +++++++++++++++
.../list/var-in-list/var-in-list.05.query.sqlpp | 22 +++++++++++++++
.../list/var-in-list/var-in-list.06.query.sqlpp | 22 +++++++++++++++
.../results/list/var-in-list/var-in-list.03.adm | 2 ++
.../results/list/var-in-list/var-in-list.04.adm | 2 ++
.../results/list/var-in-list/var-in-list.05.adm | 2 ++
.../results/list/var-in-list/var-in-list.06.adm | 2 ++
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 ++++
13 files changed, 174 insertions(+), 12 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceEnforcedListTypeRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceEnforcedListTypeRule.java
index 72f24b0..723fff2 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceEnforcedListTypeRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceEnforcedListTypeRule.java
@@ -64,6 +64,7 @@ public class IntroduceEnforcedListTypeRule implements IAlgebraicRewriteRule {
* rewrite list constructor types for list constructor functions
*/
List<Mutable<ILogicalExpression>> expressions;
+ // TODO(ali): what about other operators that could have list expressions?
switch (op.getOperatorTag()) {
case ASSIGN:
AbstractAssignOperator assignOp = (AbstractAssignOperator) op;
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
index ac2460e..e96a192 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
@@ -69,7 +69,7 @@ import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
*
* @author yingyib
*/
-public class StaticTypeCastUtil {
+public final class StaticTypeCastUtil {
private StaticTypeCastUtil() {
}
@@ -82,10 +82,11 @@ public class StaticTypeCastUtil {
* rule: @ IntroduceEnforcedListTypeRule.
*
* @param funcExpr
- * record constructor function expression
+ * a function expression
* @param reqType
- * required record type
+ * required (list) type (for when the funcExpr is a list constructor)
* @param inputType
+ * inferred (list) type (for when the funcExpr is a list constructor)
* @param env
* type environment
* @throws AlgebricksException
@@ -136,7 +137,7 @@ public class StaticTypeCastUtil {
* @param reqType
* the required type inferred from parent operators/expressions
* @param inputType
- * the current inferred
+ * the current inferred type
* @param env
* the type environment
* @return true if the type is casted; otherwise, false.
@@ -162,6 +163,7 @@ public class StaticTypeCastUtil {
return rewriteListFuncExpr(funcExpr, (AbstractCollectionType) reqType, (AbstractCollectionType) inputType,
env);
} else if (inputType.getTypeTag().equals(ATypeTag.OBJECT)) {
+ // TODO(ali): inputType? shouldn't we check against the funcExpr whether it's a record constructor?
if (reqType.equals(BuiltinType.ANY)) {
reqType = DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
}
@@ -186,13 +188,14 @@ public class StaticTypeCastUtil {
}
/**
- * only called when funcExpr is record constructor
+ * Only called when funcExpr's inferred type (the inputRecordType) is a record.
*
* @param funcExpr
- * record constructor function expression
+ * a function expression that produces a record
* @param requiredRecordType
* required record type
* @param inputRecordType
+ * inferred record type of funcExpr
* @param env
* type environment
* @throws AlgebricksException
@@ -213,13 +216,14 @@ public class StaticTypeCastUtil {
}
/**
- * only called when funcExpr is list constructor
+ * Only called when funcExpr is list constructor.
*
* @param funcExpr
* list constructor function expression
* @param requiredListType
* required list type
* @param inputListType
+ * inferred list type
* @param env
* type environment
* @throws AlgebricksException
@@ -247,19 +251,22 @@ public class StaticTypeCastUtil {
case FUNCTION_CALL:
ScalarFunctionCallExpression argFunc = (ScalarFunctionCallExpression) arg;
changed |= rewriteFuncExpr(argFunc, requiredItemType, currentItemType, env);
- changed |= castItem(argRef, argFunc, requiredItemType, env);
+ changed |= castItem(argRef, requiredItemType, env);
break;
case VARIABLE:
- // TODO(ali): why are we always casting to an open type without considering "requiredItemType"?
- changed |= injectCastToRelaxType(argRef, currentItemType, env);
+ changed |= castItem(argRef, requiredItemType, env);
+ break;
+ case CONSTANT:
+ // TODO(ali): should the constant be handled (i.e. constant array or record)?
break;
}
}
return changed;
}
- private static boolean castItem(Mutable<ILogicalExpression> itemExprRef, ScalarFunctionCallExpression itemExpr,
- IAType requiredItemType, IVariableTypeEnvironment env) throws AlgebricksException {
+ private static boolean castItem(Mutable<ILogicalExpression> itemExprRef, IAType requiredItemType,
+ IVariableTypeEnvironment env) throws AlgebricksException {
+ ILogicalExpression itemExpr = itemExprRef.getValue();
IAType itemType = (IAType) env.getType(itemExpr);
if (TypeResolverUtil.needsCast(requiredItemType, itemType) && !satisfied(requiredItemType, itemType)) {
injectCastFunction(FunctionUtil.getFunctionInfo(BuiltinFunctions.CAST_TYPE), requiredItemType, itemType,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.01.ddl.sqlpp
new file mode 100644
index 0000000..5b2b740
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.01.ddl.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE t1 AS {id: int};
+CREATE DATASET ds1(t1) primary key id;
+
+CREATE FUNCTION fun1(r) {
+ (SELECT VALUE x FROM [r] AS x)[0]
+};
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.02.update.sqlpp
new file mode 100644
index 0000000..7535183
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.02.update.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+INSERT INTO ds1 {"id": 1, "f1": "foo"};
+INSERT INTO ds1 {"id": 2, "f1": "bar"};
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.03.query.sqlpp
new file mode 100644
index 0000000..b9a79b6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.03.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+FROM ds1 AS v SELECT [v] AS list ORDER BY list;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.04.query.sqlpp
new file mode 100644
index 0000000..d61e5be
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.04.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+FROM ds1 AS s SELECT fun1(s) AS list ORDER BY list;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.05.query.sqlpp
new file mode 100644
index 0000000..42c7031
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.05.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+FROM [{'a':5}, {'a':7}] AS s SELECT fun1(s) AS list ORDER BY list;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.06.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.06.query.sqlpp
new file mode 100644
index 0000000..924b4b8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/var-in-list/var-in-list.06.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+FROM [{'a':5}, {'b':7}] AS s SELECT fun1(s) AS list ORDER BY list;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/list/var-in-list/var-in-list.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/list/var-in-list/var-in-list.03.adm
new file mode 100644
index 0000000..17ad9c9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/list/var-in-list/var-in-list.03.adm
@@ -0,0 +1,2 @@
+{ "list": [ { "id": 2, "f1": "bar" } ] }
+{ "list": [ { "id": 1, "f1": "foo" } ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/list/var-in-list/var-in-list.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/list/var-in-list/var-in-list.04.adm
new file mode 100644
index 0000000..b875625
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/list/var-in-list/var-in-list.04.adm
@@ -0,0 +1,2 @@
+{ "list": { "id": 2, "f1": "bar" } }
+{ "list": { "id": 1, "f1": "foo" } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/list/var-in-list/var-in-list.05.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/list/var-in-list/var-in-list.05.adm
new file mode 100644
index 0000000..4721f1c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/list/var-in-list/var-in-list.05.adm
@@ -0,0 +1,2 @@
+{ "list": { "a": 5 } }
+{ "list": { "a": 7 } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/list/var-in-list/var-in-list.06.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/list/var-in-list/var-in-list.06.adm
new file mode 100644
index 0000000..ff72349
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/list/var-in-list/var-in-list.06.adm
@@ -0,0 +1,2 @@
+{ "list": { "a": 5 } }
+{ "list": { "b": 7 } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index ce1303b..af6ca78 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -6140,6 +6140,11 @@
<output-dir compare="Text">query-ASTERIXDB-159-3</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="var-in-list">
+ <output-dir compare="Text">var-in-list</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="misc">
<test-case FilePath="misc">
[asterixdb] 15/23: [NO ISSUE][MD] Ensure metadata node is
initialized
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 294e5ed400c12ccb0da12c6339b1948981915840
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Wed Mar 10 01:06:17 2021 +0300
[NO ISSUE][MD] Ensure metadata node is initialized
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Before attempting to being any metadata transaction, ensure
the metadata node has been bound to avoid NPE.
Change-Id: Ic54314e9da42b4111a1aab53d1ec355ad13922e1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10444
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
(cherry picked from commit e8428051a7550578e2b28f5987e87ede41c83832)
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12023
---
.../src/main/java/org/apache/asterix/metadata/MetadataManager.java | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
index e5ee163..362ead8 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
@@ -129,6 +129,11 @@ public abstract class MetadataManager implements IMetadataManager {
@Override
public MetadataTransactionContext beginTransaction() throws RemoteException {
+ try {
+ INSTANCE.init();
+ } catch (HyracksDataException e) {
+ throw new ACIDException(e);
+ }
TxnId txnId = createTxnId();
metadataNode.beginTransaction(txnId);
return new MetadataTransactionContext(txnId);
[asterixdb] 05/23: [NO ISSUE] S3 external datasets: add support to
anonymous authentication
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0a44dbc96b4be8b160e281e5700a9fbf699cecfb
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Mon Jun 7 04:46:17 2021 +0300
[NO ISSUE] S3 external datasets: add support to anonymous authentication
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Added support to creating external datasets with no
credentials (anonymous).
Change-Id: I610f5b82d008a661946ae05bb4582d74b4e7161d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11804
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
.../aws/s3/anonymous_no_auth/test.000.ddl.sqlpp | 36 ++++++++++++++++++++++
.../aws/s3/anonymous_no_auth/test.001.ddl.sqlpp | 36 ++++++++++++++++++++++
.../aws/s3/anonymous_no_auth/test.002.ddl.sqlpp | 35 +++++++++++++++++++++
.../aws/s3/anonymous_no_auth/test.099.ddl.sqlpp | 20 ++++++++++++
.../runtimets/testsuite_external_dataset.xml | 7 +++++
.../asterix/common/exceptions/ErrorCode.java | 1 +
.../src/main/resources/asx_errormsg/en.properties | 1 +
.../asterix/external/util/ExternalDataUtils.java | 36 +++++++++++++++++++---
8 files changed, 168 insertions(+), 4 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/anonymous_no_auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/anonymous_no_auth/test.000.ddl.sqlpp
new file mode 100644
index 0000000..13cfe8a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/anonymous_no_auth/test.000.ddl.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING S3 (
+("accessKeyId"="dummyAccessKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/anonymous_no_auth/test.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/anonymous_no_auth/test.001.ddl.sqlpp
new file mode 100644
index 0000000..b8d0945
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/anonymous_no_auth/test.001.ddl.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING S3 (
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/anonymous_no_auth/test.002.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/anonymous_no_auth/test.002.ddl.sqlpp
new file mode 100644
index 0000000..9eda057
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/anonymous_no_auth/test.002.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING S3 (
+("region"="us-west-2"),
+("serviceEndpoint"="http://localhost:8001"),
+("container"="playground"),
+("definition"="json-data"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/anonymous_no_auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/anonymous_no_auth/test.099.ddl.sqlpp
new file mode 100644
index 0000000..548e632
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/anonymous_no_auth/test.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
index 096e511..43408c9 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
@@ -256,5 +256,12 @@
<output-dir compare="Text">aws/s3/include-exclude/include-12</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset">
+ <compilation-unit name="aws/s3/anonymous_no_auth">
+ <output-dir compare="Text">anonymous_no_auth</output-dir>
+ <expected-error>ASX3119: Parameter 'secretAccessKey' is required if 'accessKeyId' is provided</expected-error>
+ <expected-error>ASX3119: Parameter 'accessKeyId' is required if 'secretAccessKey' is provided</expected-error>
+ </compilation-unit>
+ </test-case>
</test-group>
</test-suite>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index a29ed3b..192113e 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -313,6 +313,7 @@ public enum ErrorCode implements IError {
INPUT_DECODE_FAILURE(3116),
FAILED_TO_PARSE_MALFORMED_LOG_RECORD(3117),
ACTIVE_ENTITY_NOT_RUNNING(3118),
+ REQUIRED_PARAM_IF_PARAM_IS_PRESENT(3119),
// Lifecycle management errors
DUPLICATE_PARTITION_ID(4000),
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 5012c7a..e3acf9c 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -316,6 +316,7 @@
3116 = Failed to decode input
3117 = Failed to parse record, malformed log record
3118 = Active Entity %1$s is not running (it is %2$s)
+3119 = Parameter '%1$s' is required if '%2$s' is provided
# Lifecycle management errors
4000 = Partition id %1$s for node %2$s already in use by node %3$s
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index bc9a038..363ec74 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.external.util;
+import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_IF_PARAM_IS_PRESENT;
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ESCAPE;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXTERNAL_SCAN_BUFFER_SIZE;
@@ -63,7 +66,9 @@ import org.apache.hyracks.dataflow.common.data.parsers.IntegerParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.LongParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.regions.Region;
@@ -623,16 +628,25 @@ public class ExternalDataUtils {
*/
public static S3Client buildAwsS3Client(Map<String, String> configuration) throws CompilationException {
// TODO(Hussain): Need to ensure that all required parameters are present in a previous step
- String accessKeyId = configuration.get(ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME);
- String secretAccessKey = configuration.get(ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME);
+ String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
+ String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
String regionId = configuration.get(ExternalDataConstants.AwsS3.REGION_FIELD_NAME);
String serviceEndpoint = configuration.get(ExternalDataConstants.AwsS3.SERVICE_END_POINT_FIELD_NAME);
S3ClientBuilder builder = S3Client.builder();
// Credentials
- AwsBasicCredentials credentials = AwsBasicCredentials.create(accessKeyId, secretAccessKey);
- builder.credentialsProvider(StaticCredentialsProvider.create(credentials));
+ AwsCredentialsProvider credentialsProvider;
+
+ // No auth required
+ if (accessKeyId == null) {
+ credentialsProvider = AnonymousCredentialsProvider.create();
+ } else {
+ credentialsProvider =
+ StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey));
+ }
+
+ builder.credentialsProvider(credentialsProvider);
builder.region(Region.of(regionId));
// Validate the service endpoint if present
@@ -667,6 +681,20 @@ public class ExternalDataUtils {
throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
}
+ // Both parameters should be passed, or neither should be passed (for anonymous/no auth)
+ String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
+ String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
+ if (accessKeyId == null || secretAccessKey == null) {
+ // If one is passed, the other is required
+ if (accessKeyId != null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, SECRET_ACCESS_KEY_FIELD_NAME,
+ ACCESS_KEY_ID_FIELD_NAME);
+ } else if (secretAccessKey != null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
+ SECRET_ACCESS_KEY_FIELD_NAME);
+ }
+ }
+
validateIncludeExclude(configuration);
// Check if the bucket is present
[asterixdb] 19/23: Merge "Merge branch 'gerrit/mad-hatter'" into
cheshire-cat
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit f2639a016afd995dac2a219eeb34936e9a8e6d21
Merge: 6db8b8e 7b90dd6
Author: Apache AsterixDB Gerrit <de...@asterixdb.apache.org>
AuthorDate: Mon Jun 28 19:06:20 2021 +0000
Merge "Merge branch 'gerrit/mad-hatter'" into cheshire-cat
.../window/misc_01/misc_01.10.query.sqlpp | 34 +++++++++++
.../results/window/misc_01/misc_01.10.adm | 4 ++
.../window/misc_01/misc_01.10.ast | 71 ++++++++++++++++++++++
.../SqlppWindowAggregationSugarVisitor.java | 7 ++-
4 files changed, 114 insertions(+), 2 deletions(-)
[asterixdb] 17/23: Merge branch 'gerrit/mad-hatter'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 7b90dd638660ba1d09c8c3f3e908b8c394c710b7
Merge: 11acb53 02a1424
Author: Murtadha Hubail <mu...@couchbase.com>
AuthorDate: Sun Jun 27 20:58:25 2021 +0300
Merge branch 'gerrit/mad-hatter'
Change-Id: I2a3075ab4c6a84fa12fce114cceae2bec46fe04a
.../window/misc_01/misc_01.10.query.sqlpp | 34 +++++++++++
.../results/window/misc_01/misc_01.10.adm | 4 ++
.../window/misc_01/misc_01.10.ast | 71 ++++++++++++++++++++++
.../SqlppWindowAggregationSugarVisitor.java | 7 ++-
4 files changed, 114 insertions(+), 2 deletions(-)
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/window/misc_01/misc_01.10.ast
index 0000000,368f261..a6cbed7
mode 000000,100644..100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/window/misc_01/misc_01.10.ast
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/window/misc_01/misc_01.10.ast
@@@ -1,0 -1,50 +1,71 @@@
+ DataverseUse test
+ Query:
+ SELECT [
+ FieldAccessor [
+ Variable [ Name=$y ]
+ Field=c2
+ ]
+ c2
+ (
+ SELECT [
- WINDOW test.nth_value@2[
- Variable [ Name=$c2 ]
++ WINDOW asterix.nth-value-impl@3[
++ (
++ SELECT ELEMENT [
++ FieldAccessor [
++ FieldAccessor [
++ Variable [ Name=#2 ]
++ Field=x
++ ]
++ Field=c2
++ ]
++ ]
++ FROM [ Variable [ Name=#1 ]
++ AS Variable [ Name=#2 ]
++ ]
++ )
+ LiteralExpr [LONG] [3]
++ FieldAccessor [
++ Variable [ Name=$x ]
++ Field=c2
++ ]
+ ]
++ AS Variable [ Name=#1 ]
++ (
++ x:=Variable [ Name=$x ]
++ )
+ OVER (
+ PARTITION BY
+ FieldAccessor [
+ Variable [ Name=$x ]
+ Field=one
+ ]
+ ORDER BY
+ FieldAccessor [
+ Variable [ Name=$x ]
+ Field=c2
+ ]
+ ASC
+ range between unbounded preceding and unbounded following exclude no others
+ )
+ nth
+ ]
+ FROM [ FunctionCall asterix.dataset@1[
+ LiteralExpr [STRING] [test.t1]
+ ]
+ AS Variable [ Name=$x ]
+ ]
+ )
+ q1
+ ]
+ FROM [ FunctionCall asterix.dataset@1[
+ LiteralExpr [STRING] [test.t1]
+ ]
+ AS Variable [ Name=$y ]
+ ]
+ Orderby
+ FieldAccessor [
+ Variable [ Name=$y ]
+ Field=c2
+ ]
+ ASC
+
[asterixdb] 10/23: [ASTERIXDB-2844][COMP] Fix syntax error with
UNNEST
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit e577fb05a502fdc046e20bd5167a47ef18fddbff
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Fri Mar 12 12:53:15 2021 -0800
[ASTERIXDB-2844][COMP] Fix syntax error with UNNEST
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Do not require AS subclause in UNNEST
- Add testcase
Change-Id: I17f4f7743e873dcb773f7a68b14660637c58aa89
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11824
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../ASTERIXDB-2844_unnest_syntax.1.query.sqlpp | 23 ++++++++++++++++++++++
.../ASTERIXDB-2844_unnest_syntax.1.adm | 4 ++++
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 +++++
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 6 +++---
4 files changed, 35 insertions(+), 3 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/unnest/ASTERIXDB-2844_unnest_syntax/ASTERIXDB-2844_unnest_syntax.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/unnest/ASTERIXDB-2844_unnest_syntax/ASTERIXDB-2844_unnest_syntax.1.query.sqlpp
new file mode 100644
index 0000000..110eefd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/unnest/ASTERIXDB-2844_unnest_syntax/ASTERIXDB-2844_unnest_syntax.1.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+with t as (select r, [r*10,r*10+1] ra from range(1, 2) r)
+select *
+from t unnest t.ra
+order by ra
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/unnest/ASTERIXDB-2844_unnest_syntax/ASTERIXDB-2844_unnest_syntax.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/unnest/ASTERIXDB-2844_unnest_syntax/ASTERIXDB-2844_unnest_syntax.1.adm
new file mode 100644
index 0000000..6b71814
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/unnest/ASTERIXDB-2844_unnest_syntax/ASTERIXDB-2844_unnest_syntax.1.adm
@@ -0,0 +1,4 @@
+{ "t": { "r": 1, "ra": [ 10, 11 ] }, "ra": 10 }
+{ "t": { "r": 1, "ra": [ 10, 11 ] }, "ra": 11 }
+{ "t": { "r": 2, "ra": [ 20, 21 ] }, "ra": 20 }
+{ "t": { "r": 2, "ra": [ 20, 21 ] }, "ra": 21 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 0015fa6..11c9634 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -12481,6 +12481,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="unnest">
+ <compilation-unit name="ASTERIXDB-2844_unnest_syntax">
+ <output-dir compare="Text">ASTERIXDB-2844_unnest_syntax</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="unnest">
<compilation-unit name="left-outer-unnest">
<output-dir compare="Text">left-outer-unnest</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index f4061e1..4c924a4 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -3499,12 +3499,12 @@ JoinClause JoinClause(JoinType joinType) throws ParseException :
UnnestClause UnnestClause(JoinType joinType) throws ParseException :
{
Token startToken = null;
- Expression rightExpr;
- VariableExpr rightVar;
+ Expression rightExpr = null;
+ VariableExpr rightVar = null;
VariableExpr posVar = null;
}
{
- (<UNNEST>|<CORRELATE>|<FLATTEN>) { startToken = token; } rightExpr = Expression() ((<AS>)? rightVar = Variable()) (<AT> posVar = Variable())?
+ (<UNNEST>|<CORRELATE>|<FLATTEN>) { startToken = token; } rightExpr = Expression() ((<AS>)? rightVar = Variable())? (<AT> posVar = Variable())?
{
if (rightVar == null) {
rightVar = ExpressionToVariableUtil.getGeneratedVariable(rightExpr, true);
[asterixdb] 11/23: [NO ISSUE][OTH] Extensible QueryServiceServlet
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 11acb536a3078f2923b4569747908c546538c27c
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Mon Jun 14 03:41:31 2021 +0300
[NO ISSUE][OTH] Extensible QueryServiceServlet
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Set StatementProperties before attempting to execute the statements
to have any changes reflected in case of statements execution failures.
- Pass execution state when a request fails.
Change-Id: Ic84aa2d7641a0f51c2d2ec13f2900066b225ec5b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11923
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../asterix/api/http/server/NCQueryServiceServlet.java | 5 +++--
.../asterix/api/http/server/QueryServiceServlet.java | 15 ++++++++-------
.../app/message/ExecuteStatementRequestMessage.java | 5 ++---
.../apache/asterix/app/translator/QueryTranslator.java | 3 ++-
4 files changed, 15 insertions(+), 13 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
index d6ebdad..2189c17 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
@@ -48,6 +48,7 @@ import org.apache.hyracks.api.exceptions.Warning;
import org.apache.hyracks.api.util.ExceptionUtils;
import org.apache.hyracks.http.api.IChannelClosedHandler;
import org.apache.hyracks.http.api.IServletRequest;
+import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.HttpServer;
import org.apache.hyracks.http.server.InterruptOnCloseHandler;
import org.apache.hyracks.ipc.exceptions.IPCException;
@@ -167,13 +168,13 @@ public class NCQueryServiceServlet extends QueryServiceServlet {
@Override
protected void handleExecuteStatementException(Throwable t, RequestExecutionState executionState,
- QueryServiceRequestParameters param) {
+ QueryServiceRequestParameters param, IServletResponse response) {
if (t instanceof TimeoutException // TODO(mblow): I don't think t can ever been an instance of TimeoutException
|| ExceptionUtils.matchingCause(t, candidate -> candidate instanceof IPCException)) {
GlobalConfig.ASTERIX_LOGGER.log(Level.WARN, t.toString(), t);
executionState.setStatus(ResultStatus.FAILED, HttpResponseStatus.SERVICE_UNAVAILABLE);
} else {
- super.handleExecuteStatementException(t, executionState, param);
+ super.handleExecuteStatementException(t, executionState, param, response);
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index a043050..abe9716 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -305,9 +305,9 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
}
errorCount = 0;
} catch (Exception | org.apache.asterix.lang.sqlpp.parser.TokenMgrError e) {
- handleExecuteStatementException(e, executionState, param);
+ handleExecuteStatementException(e, executionState, param, response);
response.setStatus(executionState.getHttpStatus());
- requestFailed(e, responsePrinter);
+ requestFailed(e, responsePrinter, executionState);
} finally {
executionState.finish();
}
@@ -419,7 +419,7 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
}
protected boolean handleIFormattedException(IError error, IFormattedException ex,
- RequestExecutionState executionState, QueryServiceRequestParameters param) {
+ RequestExecutionState executionState, QueryServiceRequestParameters param, IServletResponse response) {
if (error instanceof ErrorCode) {
switch ((ErrorCode) error) {
case INVALID_REQ_PARAM_VAL:
@@ -451,7 +451,7 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
}
protected void handleExecuteStatementException(Throwable t, RequestExecutionState executionState,
- QueryServiceRequestParameters param) {
+ QueryServiceRequestParameters param, IServletResponse response) {
if (t instanceof org.apache.asterix.lang.sqlpp.parser.TokenMgrError || t instanceof AlgebricksException) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("handleException: {}: {}", t.getMessage(), LogRedactionUtil.statement(param.toString()),
@@ -465,8 +465,8 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
} else if (t instanceof IFormattedException) {
IFormattedException formattedEx = (IFormattedException) t;
Optional<IError> maybeError = formattedEx.getError();
- if (maybeError.isPresent()
- && handleIFormattedException(maybeError.get(), (IFormattedException) t, executionState, param)) {
+ if (maybeError.isPresent() && handleIFormattedException(maybeError.get(), (IFormattedException) t,
+ executionState, param, response)) {
return;
}
}
@@ -496,7 +496,8 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, param.isCSVWithHeader());
}
- protected void requestFailed(Throwable throwable, ResponsePrinter responsePrinter) {
+ protected void requestFailed(Throwable throwable, ResponsePrinter responsePrinter,
+ RequestExecutionState executionState) {
final ExecutionError executionError = ExecutionError.of(throwable);
responsePrinter.addResultPrinter(new ErrorsPrinter(Collections.singletonList(executionError)));
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
index 29ee76d..30b98ec 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
@@ -139,6 +139,8 @@ public class ExecuteStatementRequestMessage implements ICcAddressedMessage {
IStorageComponentProvider storageComponentProvider = ccAppCtx.getStorageComponentProvider();
IStatementExecutorFactory statementExecutorFactory = ccApp.getStatementExecutorFactory();
ExecuteStatementResponseMessage responseMsg = new ExecuteStatementResponseMessage(requestMessageId);
+ final IStatementExecutor.StatementProperties statementProperties = new IStatementExecutor.StatementProperties();
+ responseMsg.setStatementProperties(statementProperties);
try {
List<Warning> warnings = new ArrayList<>();
IParser parser = compilationProvider.getParserFactory().createParser(statementsText);
@@ -160,8 +162,6 @@ public class ExecuteStatementRequestMessage implements ICcAddressedMessage {
compilationProvider, storageComponentProvider, new ResponsePrinter(sessionOutput));
final IStatementExecutor.Stats stats = new IStatementExecutor.Stats();
stats.setProfileType(profileType);
- final IStatementExecutor.StatementProperties statementProperties =
- new IStatementExecutor.StatementProperties();
Map<String, IAObject> stmtParams = RequestParameters.deserializeParameterValues(statementParameters);
final IRequestParameters requestParameters =
new RequestParameters(requestReference, statementsText, null, resultProperties, stats,
@@ -174,7 +174,6 @@ public class ExecuteStatementRequestMessage implements ICcAddressedMessage {
responseMsg.setResult(outWriter.toString());
responseMsg.setMetadata(outMetadata);
responseMsg.setStats(stats);
- responseMsg.setStatementProperties(statementProperties);
responseMsg.setExecutionPlans(translator.getExecutionPlans());
responseMsg.setWarnings(warnings);
} catch (AlgebricksException | HyracksException | org.apache.asterix.lang.sqlpp.parser.TokenMgrError pe) {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 4b6f01d..38225d7 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -3930,7 +3930,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
appCtx.getRequestTracker().track(clientRequest);
}
- protected void validateStatements(IRequestParameters requestParameters) throws CompilationException {
+ protected void validateStatements(IRequestParameters requestParameters)
+ throws CompilationException, HyracksDataException {
validateStatements(statements, requestParameters.isMultiStatement(),
requestParameters.getStatementCategoryRestrictionMask());
}
[asterixdb] 03/23: [ASTERIXDB-2732][FUN] Support missing and union
type in get_object_field_value
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit bba56a5da434a77b63e856a353ea7e7a4455ef7d
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Thu Jun 3 20:50:08 2021 +0300
[ASTERIXDB-2732][FUN] Support missing and union type in get_object_field_value
- user model changes: no
- storage format changes: no
- interface changes: no
Change-Id: I84355e0ae91afd0328e636aa4f3bed6eeb44e055
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11764
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
---
...cess.1.query.sqlpp => field-access.1.ddl.sqlpp} | 19 ++++++++---
...s.1.query.sqlpp => field-access.2.update.sqlpp} | 8 ++---
...ss.1.query.sqlpp => field-access.3.query.sqlpp} | 10 +++---
...ss.1.query.sqlpp => field-access.4.query.sqlpp} | 9 ++---
.../null-missing/field-access/field-access.1.adm | 4 ++-
.../null-missing/field-access/field-access.2.adm | 3 ++
.../records/GetRecordFieldValueEvalFactory.java | 39 ++++++++++++++--------
7 files changed, 61 insertions(+), 31 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.1.ddl.sqlpp
similarity index 73%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.1.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.1.ddl.sqlpp
index 7be4593..0d6ab35 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.1.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.1.ddl.sqlpp
@@ -17,7 +17,18 @@
* under the License.
*/
-{
- 'null': {'a': null}.a,
- 'missing': {'b': missing}.b
-};
\ No newline at end of file
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop dataset samptable if exists;
+drop type samptabletype if exists;
+
+create type samptabletype as closed {
+ id: tinyint,
+ mf: tinyint?,
+ rf: { pid:tinyint}?
+};
+
+
+create dataset samptable(samptabletype) primary key id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.2.update.sqlpp
similarity index 80%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.1.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.2.update.sqlpp
index 7be4593..8a7e88b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.1.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.2.update.sqlpp
@@ -17,7 +17,7 @@
* under the License.
*/
-{
- 'null': {'a': null}.a,
- 'missing': {'b': missing}.b
-};
\ No newline at end of file
+use test;
+insert into samptable ({'id' : 0, 'mf':missing, 'rf':{'pid':4}});
+insert into samptable ({'id' : 1, 'mf':null, 'rf':missing});
+insert into samptable ({'id' : 2, 'mf':5, 'rf':null});
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.3.query.sqlpp
similarity index 82%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.1.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.3.query.sqlpp
index 7be4593..bed3710 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.1.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.3.query.sqlpp
@@ -17,7 +17,9 @@
* under the License.
*/
-{
- 'null': {'a': null}.a,
- 'missing': {'b': missing}.b
-};
\ No newline at end of file
+use test;
+
+from samptable d
+select id, get_object_field_value(d, 'rf') as field_value,
+ get_object_field_value(d, 'rf') is missing as is_missing
+order by id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.4.query.sqlpp
similarity index 82%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.1.query.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.4.query.sqlpp
index 7be4593..553dc55 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.1.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/null-missing/field-access/field-access.4.query.sqlpp
@@ -16,8 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
+use test;
-{
- 'null': {'a': null}.a,
- 'missing': {'b': missing}.b
-};
\ No newline at end of file
+from samptable d
+select id, get_object_field_value(d, 'mf') as field_value,
+ get_object_field_value(d, 'mf') is missing as is_missing
+order by id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/field-access/field-access.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/field-access/field-access.1.adm
index d7a20ef..f157d67 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/field-access/field-access.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/field-access/field-access.1.adm
@@ -1 +1,3 @@
-{ "null": null }
+{ "id": 0, "field_value": { "pid": 4 }, "is_missing": false }
+{ "id": 1, "is_missing": true }
+{ "id": 2, "field_value": null, "is_missing": false }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/field-access/field-access.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/field-access/field-access.2.adm
new file mode 100644
index 0000000..767c19b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/null-missing/field-access/field-access.2.adm
@@ -0,0 +1,3 @@
+{ "id": 0, "is_missing": true }
+{ "id": 1, "field_value": null, "is_missing": false }
+{ "id": 2, "field_value": 5, "is_missing": false }
\ No newline at end of file
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/GetRecordFieldValueEvalFactory.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/GetRecordFieldValueEvalFactory.java
index 103a164..d6c0e21 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/GetRecordFieldValueEvalFactory.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/GetRecordFieldValueEvalFactory.java
@@ -27,7 +27,9 @@ import org.apache.asterix.formats.nontagged.BinaryHashFunctionFactoryProvider;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.types.runtime.RuntimeRecordTypeInfo;
import org.apache.asterix.om.utils.NonTaggedFormatUtil;
import org.apache.asterix.om.utils.RecordUtil;
@@ -93,7 +95,6 @@ public class GetRecordFieldValueEvalFactory implements IScalarEvaluatorFactory {
if (PointableHelper.checkAndSetMissingOrNull(result, inputArg0, inputArg1)) {
return;
}
-
byte[] serFldName = inputArg1.getByteArray();
int serFldNameOffset = inputArg1.getStartOffset();
int serFldNameLen = inputArg1.getLength();
@@ -114,22 +115,32 @@ public class GetRecordFieldValueEvalFactory implements IScalarEvaluatorFactory {
int subFieldIndex = recTypeInfo.getFieldIndex(serFldName, serFldNameOffset + 1, serFldNameLen - 1);
if (subFieldIndex >= 0) {
int nullBitmapSize = RecordUtil.computeNullBitmapSize(recordType);
+
subFieldOffset = ARecordSerializerDeserializer.getFieldOffsetById(serRecord, serRecordOffset,
subFieldIndex, nullBitmapSize, recordType.isOpen());
- if (subFieldOffset == 0) {
- // the field is null, we checked the null bit map
- out.writeByte(ATypeTag.SERIALIZED_NULL_TYPE_TAG);
- result.set(resultStorage);
- return;
+ switch (subFieldOffset) {
+ case -1:
+ out.writeByte(ATypeTag.SERIALIZED_MISSING_TYPE_TAG);
+ result.set(resultStorage);
+ return;
+ case 0:
+ out.writeByte(ATypeTag.SERIALIZED_NULL_TYPE_TAG);
+ result.set(resultStorage);
+ return;
+ default:
+ IAType fieldType = recordType.getFieldTypes()[subFieldIndex];
+ if (fieldType.getTypeTag() == ATypeTag.UNION) {
+ fieldType = ((AUnionType) fieldType).getActualType();
+ }
+ ATypeTag fieldTypeTag = fieldType.getTypeTag();
+ subFieldLength = NonTaggedFormatUtil.getFieldValueLength(serRecord, subFieldOffset,
+ fieldTypeTag, false);
+ // write result.
+ out.writeByte(fieldTypeTag.serialize());
+ out.write(serRecord, subFieldOffset, subFieldLength);
+ result.set(resultStorage);
+ return;
}
- ATypeTag fieldTypeTag = recordType.getFieldTypes()[subFieldIndex].getTypeTag();
- subFieldLength =
- NonTaggedFormatUtil.getFieldValueLength(serRecord, subFieldOffset, fieldTypeTag, false);
- // write result.
- out.writeByte(fieldTypeTag.serialize());
- out.write(serRecord, subFieldOffset, subFieldLength);
- result.set(resultStorage);
- return;
}
// Look at open fields.
[asterixdb] 22/23: Merge branch 'gerrit/cheshire-cat' into 'master'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 9d6c228d791ceb99295960c235b2fc9b9e9aabeb
Merge: 86c1812 5daf5ea
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Tue Jun 29 15:27:09 2021 -0400
Merge branch 'gerrit/cheshire-cat' into 'master'
Change-Id: I48f090a23150281b4841d4e1e45d9b004d22d60e
.../optimizer/rules/PushGroupByThroughProduct.java | 17 +++---
.../api/http/server/NCQueryServiceServlet.java | 5 +-
.../api/http/server/QueryServiceServlet.java | 15 ++---
.../message/ExecuteStatementRequestMessage.java | 5 +-
.../asterix/app/translator/QueryTranslator.java | 3 +-
.../gby-cross-join/gby-cross-join.1.ddl.sqlpp | 34 +++++++++++
.../gby-cross-join/gby-cross-join.2.update.sqlpp | 34 +++++++++++
.../gby-cross-join/gby-cross-join.3.query.sqlpp | 25 ++++++++
.../window/misc_01/misc_01.10.query.sqlpp | 34 +++++++++++
.../group-by/gby-cross-join/gby-cross-join.3.adm | 2 +
.../results/window/misc_01/misc_01.10.adm | 4 ++
.../window/misc_01/misc_01.10.ast | 71 ++++++++++++++++++++++
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 ++
.../SqlppWindowAggregationSugarVisitor.java | 7 ++-
asterixdb/pom.xml | 4 +-
15 files changed, 241 insertions(+), 24 deletions(-)