You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by xi...@apache.org on 2018/12/18 20:54:08 UTC
asterixdb git commit: [ASTERIXDB-2495][ING] Avoid LET in applying
functions to feeds
Repository: asterixdb
Updated Branches:
refs/heads/master a00d03954 -> c7c8fffdf
[ASTERIXDB-2495][ING] Avoid LET in applying functions to feeds
- user model changes: no
- storage format changes: no
- interface changes: no
LET is not necessary for applying functions to data feeds. We could
inline the function calls when constructing the pipeline query.
Change-Id: I65842f9ac84891b363d7e0a02425258d0df794e7
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3072
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/c7c8fffd
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/c7c8fffd
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/c7c8fffd
Branch: refs/heads/master
Commit: c7c8fffdf11be2d91b3e070cd302c87b12e6aa87
Parents: a00d039
Author: Xikui Wang <xk...@gmail.com>
Authored: Sat Dec 15 05:33:29 2018 +0800
Committer: Xikui Wang <xk...@gmail.com>
Committed: Tue Dec 18 12:53:22 2018 -0800
----------------------------------------------------------------------
.../apache/asterix/utils/FeedOperations.java | 23 +++-----
.../udf_filter_on_feed.1.ddl.sqlpp | 46 +++++++++++++++
.../udf_filter_on_feed.2.update.sqlpp | 20 +++++++
.../udf_filter_on_feed.3.lib.sqlpp | 19 +++++++
.../udf_filter_on_feed.4.update.sqlpp | 25 ++++++++
.../udf_filter_on_feed.5.query.sqlpp | 22 +++++++
.../udf_filter_on_feed/udf_filter_on_feed.1.adm | 1 +
.../validate-default-library.1.adm | 1 +
.../external/library/WordInListFactory.java | 31 ++++++++++
.../external/library/WordInListFunction.java | 60 ++++++++++++++++++++
.../src/test/resources/library_descriptor.xml | 8 +++
11 files changed, 240 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c7c8fffd/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
index 593d7ce..d22e929 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
@@ -184,8 +184,8 @@ public class FeedOperations {
argExprs.add(new LiteralExpr(new IntegerLiteral((Integer) arg)));
} else if (arg instanceof String) {
argExprs.add(new LiteralExpr(new StringLiteral((String) arg)));
- } else if (arg instanceof VariableExpr) {
- argExprs.add((VariableExpr) arg);
+ } else if (arg instanceof Expression) {
+ argExprs.add((Expression) arg);
}
}
return argExprs;
@@ -216,25 +216,16 @@ public class FeedOperations {
whereClause = new WhereClause(whereClauseQuery.getBody());
}
- // TODO: This can be the place to add select predicate for ingestion
// Attaching functions
- int varIdx = 1;
- VariableExpr previousVarExpr = fromTermLeftExpr;
- ArrayList<LetClause> letClauses = new ArrayList<>();
- for (FunctionSignature funcSig : feedConnection.getAppliedFunctions()) {
- VarIdentifier intermediateVar = SqlppVariableUtil
- .toInternalVariableIdentifier(FEED_DATAFLOW_INTERMEIDATE_VAL_PREFIX + String.valueOf(varIdx));
- VariableExpr intermediateVarExpr = new VariableExpr(intermediateVar);
- CallExpr functionCallExpr = new CallExpr(funcSig, addArgs(previousVarExpr));
- previousVarExpr = intermediateVarExpr;
- LetClause letClause = new LetClause(intermediateVarExpr, functionCallExpr);
- letClauses.add(letClause);
- varIdx++;
+ Expression previousVarExpr = fromTermLeftExpr;
+ for (FunctionSignature functionSignature : feedConnection.getAppliedFunctions()) {
+ CallExpr functionCallExpr = new CallExpr(functionSignature, addArgs(previousVarExpr));
+ previousVarExpr = functionCallExpr;
}
// Constructing select clause
SelectElement selectElement = new SelectElement(previousVarExpr);
SelectClause selectClause = new SelectClause(selectElement, null, false);
- SelectBlock selectBlock = new SelectBlock(selectClause, fromClause, letClauses, whereClause, null, null, null);
+ SelectBlock selectBlock = new SelectBlock(selectClause, fromClause, null, whereClause, null, null, null);
SelectSetOperation selectSetOperation = new SelectSetOperation(new SetOperationInput(selectBlock, null), null);
SelectExpression body = new SelectExpression(null, selectSetOperation, null, null, true);
Query query = new Query(false, true, body, 0);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c7c8fffd/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.1.ddl.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.1.ddl.sqlpp
new file mode 100644
index 0000000..3f3495c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.1.ddl.sqlpp
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type InputRecordType as closed {
+ id:int64,
+ fname:string,
+ lname:string,
+ age:int64,
+ dept:string
+};
+
+create type DetectResultType as {
+ id: int64,
+ sensitive: boolean
+};
+
+create dataset Results(DetectResultType) primary key id;
+
+create feed EmployeeFeed with {
+ "adapter-name" : "localfs",
+ "path" : "asterix_nc1://data/names.adm",
+ "type-name" : "InputRecordType",
+ "format" : "delimited-text",
+ "delimiter" : "|",
+ "insert-feed": "true"
+};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c7c8fffd/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.2.update.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.2.update.sqlpp
new file mode 100644
index 0000000..f58048f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.2.update.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c7c8fffd/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.3.lib.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.3.lib.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.3.lib.sqlpp
new file mode 100644
index 0000000..dbdfe16
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.3.lib.sqlpp
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+install test testlib target/data/externallib/asterix-external-data-testlib.zip
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c7c8fffd/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.4.update.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.4.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.4.update.sqlpp
new file mode 100644
index 0000000..cdca69b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.4.update.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+connect feed EmployeeFeed to dataset Results APPLY FUNCTION testlib#fnameDetector WHERE testlib#wordDetector(fname) = TRUE;
+
+START FEED EmployeeFeed;
+
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c7c8fffd/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.5.query.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.5.query.sqlpp
new file mode 100644
index 0000000..9626f36
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/udf_filter_on_feed/udf_filter_on_feed.5.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select count(*) from Results;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c7c8fffd/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/udf_filter_on_feed/udf_filter_on_feed.1.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/udf_filter_on_feed/udf_filter_on_feed.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/udf_filter_on_feed/udf_filter_on_feed.1.adm
new file mode 100644
index 0000000..e440e5c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/udf_filter_on_feed/udf_filter_on_feed.1.adm
@@ -0,0 +1 @@
+3
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c7c8fffd/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/validate-default-library/validate-default-library.1.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/validate-default-library/validate-default-library.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/validate-default-library/validate-default-library.1.adm
index a306a90..13de599 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/validate-default-library/validate-default-library.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/validate-default-library/validate-default-library.1.adm
@@ -9,3 +9,4 @@
{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#parseTweet", "Arity": "1", "Params": [ "TweetInputType" ], "ReturnType": "TweetOutputType", "Definition": "org.apache.asterix.external.library.ParseTweetFactory", "Language": "JAVA", "Kind": "SCALAR", "Dependencies": [ [ ], [ ] ] } }
{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#toUpper", "Arity": "1", "Params": [ "TextType" ], "ReturnType": "TextType", "Definition": "org.apache.asterix.external.library.UpperCaseFactory", "Language": "JAVA", "Kind": "SCALAR", "Dependencies": [ [ ], [ ] ] } }
{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#typeValidation", "Arity": "11", "Params": [ "AINT32", "AFLOAT", "ASTRING", "ADouble", "ABoolean", "APoint", "ADate", "ADatetime", "ALine", "ACircle", "ARectangle" ], "ReturnType": "AString", "Definition": "org.apache.asterix.external.library.TypeValidationFunctionFactory", "Language": "JAVA", "Kind": "SCALAR", "Dependencies": [ [ ], [ ] ] } }
+{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#wordDetector", "Arity": "1", "Params": [ "ASTRING" ], "ReturnType": "ABOOLEAN", "Definition": "org.apache.asterix.external.library.WordInListFactory", "Language": "JAVA", "Kind": "SCALAR", "Dependencies": [ [ ], [ ] ] } }
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c7c8fffd/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/WordInListFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/WordInListFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/WordInListFactory.java
new file mode 100644
index 0000000..6f87c10
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/WordInListFactory.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.external.library;
+
+import org.apache.asterix.external.api.IExternalScalarFunction;
+import org.apache.asterix.external.api.IFunctionFactory;
+
+public class WordInListFactory implements IFunctionFactory {
+
+ @Override
+ public IExternalScalarFunction getExternalFunction() {
+ return new WordInListFunction();
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c7c8fffd/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/WordInListFunction.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/WordInListFunction.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/WordInListFunction.java
new file mode 100644
index 0000000..8295fb2
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/WordInListFunction.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.external.library;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.external.api.IExternalScalarFunction;
+import org.apache.asterix.external.api.IFunctionHelper;
+import org.apache.asterix.external.library.java.base.JBoolean;
+import org.apache.asterix.external.library.java.base.JString;
+
+public class WordInListFunction implements IExternalScalarFunction {
+
+ private ArrayList<String> keywordsList;
+ private String dictPath;
+ private List<String> functionParameters;
+
+ @Override
+ public void evaluate(IFunctionHelper functionHelper) throws Exception {
+ JString input = (JString) functionHelper.getArgument(0);
+ JBoolean output = (JBoolean) functionHelper.getResultObject();
+ String fieldValue = input.getValue();
+ boolean contains = keywordsList.contains(fieldValue);
+ output.setValue(contains);
+ functionHelper.setResult(output);
+ }
+
+ @Override
+ public void initialize(IFunctionHelper functionHelper) throws Exception {
+ keywordsList = new ArrayList<>();
+ functionParameters = functionHelper.getParameters();
+ dictPath = functionParameters.get(0);
+ Files.lines(Paths.get(dictPath)).forEach(keyword -> keywordsList.add(keyword));
+ }
+
+ @Override
+ public void deinitialize() {
+ // no op
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c7c8fffd/asterixdb/asterix-external-data/src/test/resources/library_descriptor.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/test/resources/library_descriptor.xml b/asterixdb/asterix-external-data/src/test/resources/library_descriptor.xml
index 45cdc35..acbc003 100644
--- a/asterixdb/asterix-external-data/src/test/resources/library_descriptor.xml
+++ b/asterixdb/asterix-external-data/src/test/resources/library_descriptor.xml
@@ -20,6 +20,14 @@
<language>JAVA</language>
<libraryFunctions>
<libraryFunction>
+ <name>wordDetector</name>
+ <function_type>SCALAR</function_type>
+ <argument_type>ASTRING</argument_type>
+ <return_type>ABOOLEAN</return_type>
+ <definition>org.apache.asterix.external.library.WordInListFactory</definition>
+ <parameters>data/external_function/KeywordsDetector_List1.txt</parameters>
+ </libraryFunction>
+ <libraryFunction>
<name>fnameDetector</name>
<function_type>SCALAR</function_type>
<argument_type>InputRecordType</argument_type>