You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2016/03/14 11:45:40 UTC
incubator-asterixdb git commit: ASTERIXDB-1333 Fix Feed Ingestion for
Dataset with Filters
Repository: incubator-asterixdb
Updated Branches:
refs/heads/master fb431ffea -> f707c0ed2
ASTERIXDB-1333 Fix Feed Ingestion for Dataset with Filters
This change fixes a bug in the compilation of feed connection
when the dataset has filters. The bug is caused by not setting
the filter expressions during initial plan generation.
Change-Id: Ic323c9a74504921b984f68790bff2d3a7140b85b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/716
Reviewed-by: Murtadha Hubail <hu...@gmail.com>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/f707c0ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/f707c0ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/f707c0ed
Branch: refs/heads/master
Commit: f707c0ed2053e62d9b862e0cef4c17f2939533aa
Parents: fb431ff
Author: Abdullah Alamoudi <ba...@gmail.com>
Authored: Mon Mar 14 10:59:52 2016 +0300
Committer: abdullah alamoudi <ba...@gmail.com>
Committed: Mon Mar 14 03:40:24 2016 -0700
----------------------------------------------------------------------
.../LangExpressionToPlanTranslator.java | 3 +-
.../feed-with-filtered-dataset.1.ddl.aql | 45 ++++++++++++++++++++
.../feed-with-filtered-dataset.2.update.aql | 27 ++++++++++++
.../feed-with-filtered-dataset.3.query.aql | 29 +++++++++++++
.../feed-with-filtered-dataset.4.ddl.aql | 25 +++++++++++
.../feed-with-filtered-dataset.1.adm | 7 +++
.../src/test/resources/runtimets/testsuite.xml | 5 +++
7 files changed, 140 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/f707c0ed/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index 28de56b..4739b71 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -398,9 +398,10 @@ class LangExpressionToPlanTranslator
break;
}
case SUBSCRIBE_FEED: {
- ILogicalOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef,
+ InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef,
varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false);
insertOp.getInputs().add(new MutableObject<ILogicalOperator>(assign));
+ insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
leafOperator = new SinkOperator();
leafOperator.getInputs().add(new MutableObject<ILogicalOperator>(insertOp));
break;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/f707c0ed/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.ddl.aql
new file mode 100644
index 0000000..260dd1c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.ddl.aql
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test feed ingestion on a filtered dataset
+ * Expected Res : Success
+ * Date : 14th Mar 2016
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type FacebookMessageType as closed {
+ message-id: int64,
+ author-id: int64,
+ in-response-to: int64?,
+ sender-location: point?,
+ message: string,
+ send-time: datetime
+}
+
+create dataset FacebookMessages(FacebookMessageType)
+primary key message-id with filter on send-time;
+
+create feed MessageFeed using push_localfs(
+("path"="asterix_nc1://data/fbm-with-send-time.adm"),
+("format"="adm"),
+("type-name"="FacebookMessageType"));
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/f707c0ed/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.2.update.aql
new file mode 100644
index 0000000..36564a3
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.2.update.aql
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test feed ingestion on a filtered dataset
+ * Expected Res : Success
+ * Date : 14th Mar 2016
+ */
+
+use dataverse test;
+set wait-for-completion-feed "true";
+connect feed MessageFeed to dataset FacebookMessages;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/f707c0ed/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.3.query.aql
new file mode 100644
index 0000000..daf4882
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.3.query.aql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test feed ingestion on a filtered dataset
+ * Expected Res : Success
+ * Date : 14th Mar 2016
+ */
+
+use dataverse test;
+
+for $m in dataset('FacebookMessages')
+where $m.send-time > datetime("2012-08-20T10:10:00")
+return $m;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/f707c0ed/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.4.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.4.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.4.ddl.aql
new file mode 100644
index 0000000..cc50807
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.4.ddl.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test feed ingestion on a filtered dataset
+ * Expected Res : Success
+ * Date : 14th Mar 2016
+ */
+
+drop dataverse test;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/f707c0ed/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.adm
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.adm b/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.adm
new file mode 100644
index 0000000..7a4df99
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.adm
@@ -0,0 +1,7 @@
+{ "message-id": 11, "author-id": 1, "in-response-to": 1, "sender-location": point("38.97,77.49"), "message": " can't stand at&t its plan is terrible", "send-time": datetime("2012-11-20T10:10:00.000Z") }
+{ "message-id": 12, "author-id": 10, "in-response-to": 6, "sender-location": point("42.26,77.76"), "message": " can't stand t-mobile its voicemail-service is OMG:(", "send-time": datetime("2012-12-20T10:10:00.000Z") }
+{ "message-id": 14, "author-id": 9, "in-response-to": 12, "sender-location": point("41.33,85.28"), "message": " love at&t its 3G is good:)", "send-time": datetime("2013-09-20T10:10:00.000Z") }
+{ "message-id": 13, "author-id": 10, "in-response-to": 4, "sender-location": point("42.77,78.92"), "message": " dislike iphone the voice-command is bad:(", "send-time": datetime("2013-08-20T10:10:00.000Z") }
+{ "message-id": 15, "author-id": 7, "in-response-to": 11, "sender-location": point("44.47,67.11"), "message": " like iphone the voicemail-service is awesome", "send-time": datetime("2014-01-20T10:10:00.000Z") }
+{ "message-id": 9, "author-id": 3, "in-response-to": 12, "sender-location": point("34.45,96.48"), "message": " love verizon its wireless is good", "send-time": datetime("2012-09-20T10:10:00.000Z") }
+{ "message-id": 10, "author-id": 1, "in-response-to": 12, "sender-location": point("42.5,70.01"), "message": " can't stand motorola the touch-screen is terrible", "send-time": datetime("2012-10-20T10:10:00.000Z") }
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/f707c0ed/asterix-app/src/test/resources/runtimets/testsuite.xml
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index b46b75b..d50a0a6 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -36,6 +36,11 @@
</compilation-unit>
</test-case> -->
<test-case FilePath="feeds">
+ <compilation-unit name="feed-with-filtered-dataset">
+ <output-dir compare="Text">feed-with-filtered-dataset</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="feeds">
<compilation-unit name="feed-push-socket">
<output-dir compare="Text">feed-push-socket</output-dir>
</compilation-unit>