You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by bu...@apache.org on 2015/10/31 00:16:48 UTC
[40/51] [partial] incubator-asterixdb git commit: SQL++ parser: 1.
refactored asterix-aql to become asterix-lang-common and asterix-lang-aql,
where the former is the common part for different languages;
2. added asterix-lang-sqlpp on top of asterix-lang-
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-jaccard-inline.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-jaccard-inline.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-jaccard-inline.sqlpp
new file mode 100644
index 0000000..ba1f784
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-jaccard-inline.sqlpp
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy self joins a dataset, Customers, based on the similarity-jaccard function of its interest lists.
+ * Customers has a keyword index on interests, and we expect the join to be transformed into an indexed nested-loop join.
+ * We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
+ * We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+ number : int32,
+ street : string,
+ city : string
+}
+
+create type test.CustomerType as
+ closed {
+ cid : int32,
+ name : string,
+ age : int32?,
+ address : AddressType?,
+ interests : [string],
+ children : [{
+ name : string,
+ age : int32?
+ }
+]
+}
+
+create table Customers(CustomerType) primary key cid;
+
+create index interests_index on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_olist-jaccard-inline.adm"
+select element {'ainterests':a.interests,'binterests':b.interests,'jacc':jacc}
+from Customers as a,
+ Customers as b
+with jacc as /*+ indexnl */ test."similarity-jaccard"(a.interests,b.interests)
+where ((jacc >= 0.700000f) and (a.cid < b.cid))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-jaccard.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-jaccard.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-jaccard.sqlpp
new file mode 100644
index 0000000..c600a84
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/olist-jaccard.sqlpp
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, Customer and Customer2, based on the similarity-jaccard function of their interest lists.
+ * Customers has a keyword index on interests, and we expect the join to be transformed into an indexed nested-loop join.
+ * We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+ number : int32,
+ street : string,
+ city : string
+}
+
+create type test.CustomerType as
+ closed {
+ cid : int32,
+ name : string,
+ age : int32?,
+ address : AddressType?,
+ interests : [string],
+ children : [{
+ name : string,
+ age : int32?
+ }
+]
+}
+
+create table Customers(CustomerType) primary key cid;
+
+create table Customers2(CustomerType) primary key cid;
+
+create index interests_index on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_olist-jaccard.adm"
+select element {'ainterests':a.interests,'binterests':b.interests}
+from Customers as a,
+ Customers2 as b
+where (( /*+ indexnl */ test."similarity-jaccard"(a.interests,b.interests) >= 0.700000f) and (a.cid < b.cid))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ulist-fuzzyeq-jaccard.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ulist-fuzzyeq-jaccard.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ulist-fuzzyeq-jaccard.sqlpp
new file mode 100644
index 0000000..5039c74
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ulist-fuzzyeq-jaccard.sqlpp
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, Customer and Customer2, based on ~= using Jaccard of their interest sets.
+ * Customers has a keyword index on interests, and we expect the join to be transformed into an indexed nested-loop join.
+ * We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+ number : int32,
+ street : string,
+ city : string
+}
+
+create type test.CustomerType as
+ closed {
+ cid : int32,
+ name : string,
+ age : int32?,
+ address : AddressType?,
+ interests : {{string}},
+ children : [{
+ name : string,
+ age : int32?
+ }
+]
+}
+
+create table Customers(CustomerType) primary key cid;
+
+create table Customers2(CustomerType) primary key cid;
+
+create index interests_index on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_ulist-fuzzyeq-jaccard.adm"
+set "simfunction" "jaccard";
+
+set "simthreshold" "0.7f";
+
+select element {'ainterests':a.interests,'binterests':b.interests}
+from Customers as a,
+ Customers2 as b
+where ((a.interests /*+ indexnl */ ~= b.interests) and (a.cid < b.cid))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ulist-jaccard-inline.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ulist-jaccard-inline.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ulist-jaccard-inline.sqlpp
new file mode 100644
index 0000000..5046dcf
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ulist-jaccard-inline.sqlpp
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy self joins a dataset, Customers, based on the similarity-jaccard function of its interest sets.
+ * Customers has a keyword index on interests, and we expect the join to be transformed into an indexed nested-loop join.
+ * We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
+ * We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+ number : int32,
+ street : string,
+ city : string
+}
+
+create type test.CustomerType as
+ closed {
+ cid : int32,
+ name : string,
+ age : int32?,
+ address : AddressType?,
+ interests : {{string}},
+ children : [{
+ name : string,
+ age : int32?
+ }
+]
+}
+
+create table Customers(CustomerType) primary key cid;
+
+create index interests_index on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_ulist-jaccard-inline.adm"
+select element {'ainterests':a.interests,'binterests':b.interests,'jacc':jacc}
+from Customers as a,
+ Customers as b
+with jacc as /*+ indexnl */ test."similarity-jaccard"(a.interests,b.interests)
+where ((jacc >= 0.700000f) and (a.cid < b.cid))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ulist-jaccard.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ulist-jaccard.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ulist-jaccard.sqlpp
new file mode 100644
index 0000000..c8c310b
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/ulist-jaccard.sqlpp
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, Customer and Customer2, based on the similarity-jaccard function of their interest sets.
+ * Customers has a keyword index on interests, and we expect the join to be transformed into an indexed nested-loop join.
+ * We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+ number : int32,
+ street : string,
+ city : string
+}
+
+create type test.CustomerType as
+ closed {
+ cid : int32,
+ name : string,
+ age : int32?,
+ address : AddressType?,
+ interests : {{string}},
+ children : [{
+ name : string,
+ age : int32?
+ }
+]
+}
+
+create table Customers(CustomerType) primary key cid;
+
+create table Customers2(CustomerType) primary key cid;
+
+create index interests_index on Customers (interests) type keyword;
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_ulist-jaccard.adm"
+select element {'ainterests':a.interests,'binterests':b.interests}
+from Customers as a,
+ Customers2 as b
+where (( /*+ indexnl */ test."similarity-jaccard"(a.interests,b.interests) >= 0.700000f) and (a.cid < b.cid))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/word-fuzzyeq-jaccard.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/word-fuzzyeq-jaccard.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/word-fuzzyeq-jaccard.sqlpp
new file mode 100644
index 0000000..dc298c0
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/word-fuzzyeq-jaccard.sqlpp
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' word tokens.
+ * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
+ * We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.CSXType as
+ closed {
+ id : int32,
+ csxid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create table CSX(CSXType) primary key id;
+
+create index keyword_index on DBLP (title) type keyword;
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_word-fuzzyeq-jaccard.adm"
+set "simfunction" "jaccard";
+
+set "simthreshold" "0.5f";
+
+select element {'atitle':a.title,'btitle':b.title}
+from DBLP as a,
+ CSX as b
+where ((test."word-tokens"(a.title) ~= test."word-tokens"(b.title)) and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/word-jaccard-inline.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/word-jaccard-inline.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/word-jaccard-inline.sqlpp
new file mode 100644
index 0000000..318b202
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/word-jaccard-inline.sqlpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy self joins a dataset, DBLP, based on the similarity-jaccard function of its titles' word tokens.
+ * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
+ * We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
+ * We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create index keyword_index on DBLP (title) type keyword;
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_word-jaccard-inline.adm"
+select element {'atitle':a.title,'btitle':b.title,'jacc':jacc}
+from DBLP as a,
+ DBLP as b
+with jacc as test."similarity-jaccard"(test."word-tokens"(a.title),test."word-tokens"(b.title))
+where ((jacc >= 0.500000f) and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/word-jaccard.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/word-jaccard.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/word-jaccard.sqlpp
new file mode 100644
index 0000000..1f07f4a
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join-noeqjoin/word-jaccard.sqlpp
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' word tokens.
+ * DBLP has a keyword index on title, and we expect the join to be transformed into an indexed nested-loop join.
+ * We expect the top-level equi join introduced because of surrogate optimization to be removed, since it is not necessary.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.CSXType as
+ closed {
+ id : int32,
+ csxid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create table CSX(CSXType) primary key id;
+
+create index keyword_index on DBLP (title) type keyword;
+
+write output to nc1:"rttest/inverted-index-join-noeqjoin_word-jaccard.adm"
+select element {'atitle':a.title,'btitle':b.title}
+from DBLP as a,
+ CSX as b
+where ((test."similarity-jaccard"(test."word-tokens"(a.title),test."word-tokens"(b.title)) >= 0.500000f) and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/issue741.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/issue741.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/issue741.sqlpp
new file mode 100644
index 0000000..f3ba2ab
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/issue741.sqlpp
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test that left-outer-join may use an available inverted index in index subtree.
+ * Issue : 741
+ * Expected Res : Success
+ * Date : 16th May 2014
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.TwitterUserType as
+{
+ screen_name : string,
+ lang : string,
+ friends_count : int32,
+ statuses_count : int32,
+ name : string,
+ followers_count : int32
+}
+
+create type test.TweetMessageType as
+{
+ tweetid : int64,
+ user : TwitterUserType,
+ sender_location : point?,
+ send_time : datetime,
+ referred_topics : {{string}},
+ message_text : string
+}
+
+create table TweetMessages(TweetMessageType) primary key tweetid;
+
+create index topicIIx on TweetMessages (referred_topics) type keyword;
+
+write output to nc1:"rttest/inverted-index-join_issue741.adm"
+select element {'tweet':t.tweetid,'similar-tweets':(
+ select element t2.tweetid
+ from TweetMessages as t2
+ with sim as test."similarity-jaccard-check"(t.referred_topics,t2.referred_topics,0.600000f)
+ where (sim[0] and (t2.tweetid != t.tweetid))
+ )}
+from TweetMessages as t
+where ((t.send_time >= test.datetime('2011-06-18T14:10:17')) and (t.send_time < test.datetime('2011-06-18T15:10:17')))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/leftouterjoin-probe-pidx-with-join-edit-distance-check-idx_01.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/leftouterjoin-probe-pidx-with-join-edit-distance-check-idx_01.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/leftouterjoin-probe-pidx-with-join-edit-distance-check-idx_01.sqlpp
new file mode 100644
index 0000000..d29180d
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/leftouterjoin-probe-pidx-with-join-edit-distance-check-idx_01.sqlpp
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.TwitterUserType as
+ closed {
+ "screen-name" : string,
+ lang : string,
+ "friends-count" : int32,
+ "statuses-count" : int32,
+ name : string,
+ "followers-count" : int32
+}
+
+create type test.TweetMessageType as
+ closed {
+ tweetid : int64,
+ user : TwitterUserType,
+ "sender-location" : point,
+ "send-time" : datetime,
+ "referred-topics" : {{string}},
+ "message-text" : string,
+ countA : int32,
+ countB : int32
+}
+
+create table TweetMessages(TweetMessageType) primary key tweetid;
+
+create index twmSndLocIx on TweetMessages ("sender-location") type rtree;
+
+create index msgCountAIx on TweetMessages (countA) type btree;
+
+create index msgCountBIx on TweetMessages (countB) type btree;
+
+create index msgKeywordIx on TweetMessages ("message-text") type keyword;
+
+create index msgNgramIx on TweetMessages ("message-text") type ngram (3);
+
+create index topicKeywordIx on TweetMessages ("referred-topics") type keyword;
+
+write output to nc1:"rttest/inverted-index-join_leftouterjoin-probe-pidx-with-join-edit-distance-check_idx_01.adm"
+select element {'tweet':{'id':t1.tweetid,'topics':t1."message-text"},'similar-tweets':(
+ select element {'id':t2.tweetid,'topics':t2."message-text"}
+ from TweetMessages as t2
+ with sim as test."edit-distance-check"(t1."message-text",t2."message-text",7)
+ where (sim[0] and (t2.tweetid != t1.tweetid))
+ order by t2.tweetid
+ )}
+from TweetMessages as t1
+where (t1.tweetid > test.int64('240'))
+order by t1.tweetid
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.sqlpp
new file mode 100644
index 0000000..5afe4b6
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/leftouterjoin-probe-pidx-with-join-jaccard-check-idx_01.sqlpp
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.TwitterUserType as
+ closed {
+ "screen-name" : string,
+ lang : string,
+ "friends-count" : int32,
+ "statuses-count" : int32,
+ name : string,
+ "followers-count" : int32
+}
+
+create type test.TweetMessageType as
+ closed {
+ tweetid : int64,
+ user : TwitterUserType,
+ "sender-location" : point,
+ "send-time" : datetime,
+ "referred-topics" : {{string}},
+ "message-text" : string,
+ countA : int32,
+ countB : int32
+}
+
+create table TweetMessages(TweetMessageType) primary key tweetid;
+
+create index twmSndLocIx on TweetMessages ("sender-location") type rtree;
+
+create index msgCountAIx on TweetMessages (countA) type btree;
+
+create index msgCountBIx on TweetMessages (countB) type btree;
+
+create index msgKeywordIx on TweetMessages ("message-text") type keyword;
+
+create index msgNgramIx on TweetMessages ("message-text") type ngram (3);
+
+create index topicKeywordIx on TweetMessages ("referred-topics") type keyword;
+
+write output to nc1:"rttest/inverted-index-join_leftouterjoin-probe-pidx-with-join-jaccard-check_idx_01.adm"
+select element {'tweet':{'id':t1.tweetid,'topics':t1."referred-topics"},'similar-tweets':(
+ select element {'id':t2.tweetid,'topics':t2."referred-topics"}
+ from TweetMessages as t2
+ with sim as test."similarity-jaccard-check"(t1."referred-topics",t2."referred-topics",0.500000f)
+ where (sim[0] and (t2.tweetid != t1.tweetid))
+ order by t2.tweetid
+ )}
+from TweetMessages as t1
+where (t1.tweetid > test.int64('240'))
+order by t1.tweetid
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-contains.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-contains.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-contains.sqlpp
new file mode 100644
index 0000000..38dccaf
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-contains.sqlpp
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Tests whether an ngram_index is applied to optimize a join query using the contains function.
+ * The index should be applied.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create index ngram_index on DBLP (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-contains.adm"
+select element {'title1':o1.title,'title2':o2.title}
+from DBLP as o1,
+ DBLP as o2
+where (test.contains(o1.title,o2.title) and (o1.id < o2.id))
+order by o1.id,o2.id
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_01.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_01.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_01.sqlpp
new file mode 100644
index 0000000..89f490c
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_01.sqlpp
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, DBLP and CSX, based on the edit-distance-check function of their authors.
+ * DBLP has a 3-gram index on authors, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.CSXType as
+ closed {
+ id : int32,
+ csxid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create table CSX(CSXType) primary key id;
+
+create index ngram_index on DBLP (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-edit-distance-check_01.adm"
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ CSX as b
+where (test."edit-distance-check"(a.authors,b.authors,3)[0] and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_02.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_02.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_02.sqlpp
new file mode 100644
index 0000000..4353249
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_02.sqlpp
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, DBLP and CSX, based on the edit-distance-check function of their authors.
+ * CSX has a 3-gram index on authors, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.CSXType as
+ closed {
+ id : int32,
+ csxid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create table CSX(CSXType) primary key id;
+
+create index ngram_index on CSX (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-edit-distance-check_02.adm"
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ CSX as b
+where (test."edit-distance-check"(a.authors,b.authors,3)[0] and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_03.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_03.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_03.sqlpp
new file mode 100644
index 0000000..91a2c8e
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_03.sqlpp
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy self joins a dataset, DBLP, based on the edit-distance-check function of its authors.
+ * DBLP has a 3-gram index on authors, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create index ngram_index on DBLP (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-edit-distance-check_03.adm"
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ DBLP as b
+where (test."edit-distance-check"(a.authors,b.authors,3)[0] and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_04.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_04.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_04.sqlpp
new file mode 100644
index 0000000..e32b243
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-check_04.sqlpp
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy self joins a dataset, DBLP, based on the edit-distance-check function of its authors.
+ * DBLP has a 3-gram index on authors, and we expect the join to be transformed into an indexed nested-loop join.
+ * We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create index ngram_index on DBLP (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-edit-distance-check_04.adm"
+select element {'arec':a,'brec':b,'ed':ed[1]}
+from DBLP as a,
+ DBLP as b
+with ed as test."edit-distance-check"(a.authors,b.authors,3)
+where (ed[0] and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-contains.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-contains.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-contains.sqlpp
new file mode 100644
index 0000000..01fffe4
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance-contains.sqlpp
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, DBLP and CSX, based on the edit-distance-contains function of their authors.
+ * DBLP has a 3-gram index on authors, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.CSXType as
+ closed {
+ id : int32,
+ csxid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create table CSX(CSXType) primary key id;
+
+create index ngram_index on DBLP (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-edit-distance-contains.adm"
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ CSX as b
+where (test."edit-distance-contains"(a.authors,b.authors,3)[0] and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_01.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_01.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_01.sqlpp
new file mode 100644
index 0000000..84842a0
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_01.sqlpp
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, DBLP and CSX, based on the edit-distance function of their authors.
+ * DBLP has a 3-gram index on authors, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.CSXType as
+ closed {
+ id : int32,
+ csxid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create table CSX(CSXType) primary key id;
+
+create index ngram_index on DBLP (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-edit-distance_01.adm"
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ CSX as b
+where ((test."edit-distance"(a.authors,b.authors) < 3) and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_02.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_02.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_02.sqlpp
new file mode 100644
index 0000000..bfd9111
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_02.sqlpp
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, DBLP and CSX, based on the edit-distance function of their authors.
+ * CSX has a 3-gram index on authors, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.CSXType as
+ closed {
+ id : int32,
+ csxid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create table CSX(CSXType) primary key id;
+
+create index ngram_index on CSX (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-edit-distance_02.adm"
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ CSX as b
+where ((test."edit-distance"(a.authors,b.authors) < 3) and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_03.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_03.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_03.sqlpp
new file mode 100644
index 0000000..c27456f
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_03.sqlpp
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy self joins a dataset, DBLP, based on the edit-distance function of its authors.
+ * DBLP has a 3-gram index on authors, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create index ngram_index on DBLP (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-edit-distance_03.adm"
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ DBLP as b
+where ((test."edit-distance"(a.authors,b.authors) < 3) and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_04.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_04.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_04.sqlpp
new file mode 100644
index 0000000..e302d06
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-edit-distance_04.sqlpp
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy self joins a dataset, DBLP, based on the edit-distance function of its authors.
+ * DBLP has a 3-gram index on authors, and we expect the join to be transformed into an indexed nested-loop join.
+ * We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create index ngram_index on DBLP (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-edit-distance_03.adm"
+select element {'arec':a,'brec':b,'ed':ed}
+from DBLP as a,
+ DBLP as b
+with ed as test."edit-distance"(a.authors,b.authors)
+where ((ed < 3) and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-edit-distance_01.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-edit-distance_01.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-edit-distance_01.sqlpp
new file mode 100644
index 0000000..124f818
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-edit-distance_01.sqlpp
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using edit distance of their authors.
+ * DBLP has a 3-gram index on authors, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.CSXType as
+ closed {
+ id : int32,
+ csxid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create table CSX(CSXType) primary key id;
+
+create index ngram_index on CSX (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-fuzzyeq-edit-distance_01.adm"
+set "simfunction" "edit-distance";
+
+set "simthreshold" "3";
+
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ CSX as b
+where ((a.authors ~= b.authors) and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-edit-distance_02.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-edit-distance_02.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-edit-distance_02.sqlpp
new file mode 100644
index 0000000..8fc2a0c
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-edit-distance_02.sqlpp
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using edit distance of their authors.
+ * CSX has a 3-gram index on authors, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.CSXType as
+ closed {
+ id : int32,
+ csxid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create table CSX(CSXType) primary key id;
+
+create index ngram_index on DBLP (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-fuzzyeq-edit-distance_01.adm"
+set "simfunction" "edit-distance";
+
+set "simthreshold" "3";
+
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ CSX as b
+where ((a.authors ~= b.authors) and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-edit-distance_03.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-edit-distance_03.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-edit-distance_03.sqlpp
new file mode 100644
index 0000000..55880e4
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-edit-distance_03.sqlpp
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy self joins a dataset, DBLP, based on ~= using edit distance of its authors.
+ * DBLP has a 3-gram index on authors, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create index ngram_index on DBLP (authors) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-fuzzyeq-edit-distance_03.adm"
+set "simfunction" "edit-distance";
+
+set "simthreshold" "3";
+
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ DBLP as b
+where ((a.authors ~= b.authors) and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-jaccard_01.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-jaccard_01.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-jaccard_01.sqlpp
new file mode 100644
index 0000000..a561118
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-jaccard_01.sqlpp
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard of their titles' 3-gram tokens.
+ * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+set "import-private-functions" "true";
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.CSXType as
+ closed {
+ id : int32,
+ csxid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create table CSX(CSXType) primary key id;
+
+create index ngram_index on DBLP (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-fuzzyeq-jaccard_01.adm"
+set "simfunction" "jaccard";
+
+set "simthreshold" "0.5f";
+
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ CSX as b
+where ((test."gram-tokens"(a.title,3,false) ~= test."gram-tokens"(b.title,3,false)) and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-jaccard_02.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-jaccard_02.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-jaccard_02.sqlpp
new file mode 100644
index 0000000..a91dd7b
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-jaccard_02.sqlpp
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, DBLP and CSX, based on ~= using Jaccard their titles' 3-gram tokens.
+ * CSX has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+set "import-private-functions" "true";
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.CSXType as
+ closed {
+ id : int32,
+ csxid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create table CSX(CSXType) primary key id;
+
+create index ngram_index on CSX (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-fuzzyeq-jaccard_02.adm"
+set "simfunction" "jaccard";
+
+set "simthreshold" "0.5f";
+
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ CSX as b
+where ((test."gram-tokens"(a.title,3,false) ~= test."gram-tokens"(b.title,3,false)) and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-jaccard_03.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-jaccard_03.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-jaccard_03.sqlpp
new file mode 100644
index 0000000..fdd7f86
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-fuzzyeq-jaccard_03.sqlpp
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy self joins a dataset, DBLP, based on ~= using Jaccard of its titles' 3-gram tokens.
+ * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+set "import-private-functions" "true";
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create index ngram_index on DBLP (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-fuzzyeq-jaccard_03.adm"
+set "simfunction" "jaccard";
+
+set "simthreshold" "0.5f";
+
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ DBLP as b
+where ((test."gram-tokens"(a.title,3,false) ~= test."gram-tokens"(b.title,3,false)) and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_01.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_01.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_01.sqlpp
new file mode 100644
index 0000000..9a3dd33
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_01.sqlpp
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard-check function of their titles' 3-gram tokens.
+ * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+set "import-private-functions" "true";
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.CSXType as
+ closed {
+ id : int32,
+ csxid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create table CSX(CSXType) primary key id;
+
+create index ngram_index on DBLP (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-jaccard-check_01.adm"
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ CSX as b
+where (test."similarity-jaccard-check"(test."gram-tokens"(a.title,3,false),test."gram-tokens"(b.title,3,false),0.500000f)[0] and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_02.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_02.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_02.sqlpp
new file mode 100644
index 0000000..9c43865
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_02.sqlpp
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, DBLP and CSX, based the similarity-jaccard-check function of their titles' 3-gram tokens.
+ * CSX has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+set "import-private-functions" "true";
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.CSXType as
+ closed {
+ id : int32,
+ csxid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create table CSX(CSXType) primary key id;
+
+create index ngram_index on CSX (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-jaccard-check_02.adm"
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ CSX as b
+where (test."similarity-jaccard-check"(test."gram-tokens"(a.title,3,false),test."gram-tokens"(b.title,3,false),0.500000f)[0] and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_03.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_03.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_03.sqlpp
new file mode 100644
index 0000000..3f3e334
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_03.sqlpp
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy self joins a dataset, DBLP, based on the similarity-jaccard-check function of its titles' 3-gram tokens.
+ * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+set "import-private-functions" "true";
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create index ngram_index on DBLP (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-jaccard-check_03.adm"
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ DBLP as b
+where (test."similarity-jaccard-check"(test."gram-tokens"(a.title,3,false),test."gram-tokens"(b.title,3,false),0.500000f)[0] and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_04.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_04.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_04.sqlpp
new file mode 100644
index 0000000..90497fc
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard-check_04.sqlpp
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy self joins a dataset, DBLP, based on the similarity-jaccard-check function of its titles' 3-gram tokens.
+ * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
+ * We test the inlining of variables that enable the select to be pushed into the join for subsequent optimization with an index.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+set "import-private-functions" "true";
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create index ngram_index on DBLP (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-jaccard-check_04.adm"
+select element {'arec':a,'brec':b,'jacc':jacc[1]}
+from DBLP as a,
+ DBLP as b
+with jacc as test."similarity-jaccard-check"(test."gram-tokens"(a.title,3,false),test."gram-tokens"(b.title,3,false),0.500000f)
+where (jacc[0] and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard_01.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard_01.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard_01.sqlpp
new file mode 100644
index 0000000..8a456c6
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard_01.sqlpp
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, DBLP and CSX, based on the similarity-jaccard function of their titles' 3-gram tokens.
+ * DBLP has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+set "import-private-functions" "true";
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.CSXType as
+ closed {
+ id : int32,
+ csxid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create table CSX(CSXType) primary key id;
+
+create index ngram_index on DBLP (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-jaccard_01.adm"
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ CSX as b
+where ((test."similarity-jaccard"(test."gram-tokens"(a.title,3,false),test."gram-tokens"(b.title,3,false)) >= 0.500000f) and (a.id < b.id))
+;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/391f09e5/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard_02.sqlpp
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard_02.sqlpp b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard_02.sqlpp
new file mode 100644
index 0000000..b8bde23
--- /dev/null
+++ b/asterix-app/src/test/resources/optimizerts/queries_sqlpp/inverted-index-join/ngram-jaccard_02.sqlpp
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Fuzzy joins two datasets, DBLP and CSX, based the similarity-jaccard function of their titles' 3-gram tokens.
+ * CSX has a 3-gram index on title, and we expect the join to be transformed into an indexed nested-loop join.
+ * Success : Yes
+ */
+
+drop database test if exists;
+create database test;
+
+use test;
+
+
+set "import-private-functions" "true";
+
+create type test.DBLPType as
+ closed {
+ id : int32,
+ dblpid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create type test.CSXType as
+ closed {
+ id : int32,
+ csxid : string,
+ title : string,
+ authors : string,
+ misc : string
+}
+
+create table DBLP(DBLPType) primary key id;
+
+create table CSX(CSXType) primary key id;
+
+create index ngram_index on CSX (title) type ngram (3);
+
+write output to nc1:"rttest/inverted-index-join_ngram-jaccard_02.adm"
+select element {'arec':a,'brec':b}
+from DBLP as a,
+ CSX as b
+where ((test."similarity-jaccard"(test."gram-tokens"(a.title,3,false),test."gram-tokens"(b.title,3,false)) >= 0.500000f) and (a.id < b.id))
+;