You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by "Anon. E. Moose (Code Review)" <do...@asterixdb.incubator.apache.org> on 2018/12/12 18:39:49 UTC

Change in asterixdb[master]: fixed regex parsing of parse-only; filter on freeVars;

Sandeep.gupta@couchbase.com has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/3077

Change subject: fixed regex parsing of parse-only; filter on freeVars;
......................................................................

fixed regex parsing of parse-only; filter on freeVars;

Change-Id: I4b59edb9d4ad759dfe81c071d50f7eaa7000bdd1
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
M asterixdb/asterix-app/src/test/resources/parseonly/queries_sqlpp/named_01.1.query.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/named_01.1.parse.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/named_01.2.parse.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/named_01.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/named_01.2.adm
M asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java
D asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/ParseValueParser.java
M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
11 files changed, 79 insertions(+), 90 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/77/3077/1

diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index 707590a..c771ca0 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -28,7 +28,13 @@
 import java.io.PrintWriter;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
-import java.util.*;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import java.util.concurrent.ConcurrentMap;
 import java.util.function.BiFunction;
 import java.util.function.Function;
@@ -40,6 +46,7 @@
 import org.apache.asterix.common.config.GlobalConfig;
 import org.apache.asterix.common.context.IStorageComponentProvider;
 import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.exceptions.CompilationException;
 import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.asterix.common.exceptions.RuntimeDataException;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
@@ -47,12 +54,22 @@
 import org.apache.asterix.lang.common.base.IParser;
 import org.apache.asterix.lang.common.base.IParserFactory;
 import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.Query;
 import org.apache.asterix.lang.sqlpp.parser.ParseOnlyResult;
 import org.apache.asterix.metadata.MetadataManager;
 import org.apache.asterix.om.base.IAObject;
-import org.apache.asterix.translator.*;
+import org.apache.asterix.translator.ExecutionPlans;
+import org.apache.asterix.translator.ExecutionPlansJsonPrintUtil;
+import org.apache.asterix.translator.IRequestParameters;
+import org.apache.asterix.translator.IStatementExecutor;
 import org.apache.asterix.translator.IStatementExecutor.ResultDelivery;
 import org.apache.asterix.translator.IStatementExecutor.Stats;
+import org.apache.asterix.translator.IStatementExecutorContext;
+import org.apache.asterix.translator.IStatementExecutorFactory;
+import org.apache.asterix.translator.ResultProperties;
+import org.apache.asterix.translator.SessionConfig;
+import org.apache.asterix.translator.SessionOutput;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.api.application.IServiceContext;
 import org.apache.hyracks.api.exceptions.HyracksException;
@@ -64,7 +81,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import org.apache.asterix.lang.sqlpp.parser.ParseValueParser;
 
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.databind.JsonMappingException;
@@ -451,6 +467,7 @@
             param.setTimeout(request.getParameter(Parameter.TIMEOUT.str()));
             param.setMaxResultReads(request.getParameter(Parameter.MAX_RESULT_READS.str()));
             param.setPlanFormat(request.getParameter(Parameter.PLAN_FORMAT.str()));
+            param.setParseOnly(Boolean.parseBoolean(request.getParameter(Parameter.PARSE_ONLY.str())));
             final String multiStatementParam = request.getParameter(Parameter.MULTI_STATEMENT.str());
             param.setMultiStatement(multiStatementParam == null || Boolean.parseBoolean(multiStatementParam));
             try {
@@ -536,7 +553,7 @@
         try {
             if (param.getStatement() == null || param.getStatement().isEmpty()) {
                 throw new RuntimeDataException(ErrorCode.NO_STATEMENT_PROVIDED);
-            } //TODO: moved this out of try-catch block.
+            }
 
             String statementsText = param.getStatement() + ";";
 
@@ -599,9 +616,17 @@
 
     protected ParseOnlyResult parseStatement(String statementsText) throws Exception {
         IParserFactory factory = compilationProvider.getParserFactory();
-        ParseValueParser parser = new ParseValueParser(factory, statementsText);
-        parser.parse();
-        return parser.getParseOnlyResult();
+        IParser parser = factory.createParser(statementsText);
+        List<Statement> stmts = parser.parse();
+        if (stmts.size() != 1) {
+            throw new CompilationException("MultiStatement not support"); //TODO: proper exception message
+        }
+        Query query = (Query) stmts.get(0);
+        Set<VariableExpr> extVars =
+                compilationProvider.getRewriterFactory().createQueryRewriter().getExternalVariables(query.getBody());
+        ParseOnlyResult parseOnlyResult = new ParseOnlyResult();
+        parseOnlyResult.setExternalVariables(extVars);
+        return parseOnlyResult;
     }
 
     protected void executeStatement(String statementsText, SessionOutput sessionOutput,
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 806eb54..1ecaa1a 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -37,7 +37,16 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -123,7 +132,7 @@
     private static final Pattern HANDLE_VARIABLE_PATTERN = Pattern.compile("handlevariable=(\\w+)");
     private static final Pattern VARIABLE_REF_PATTERN = Pattern.compile("\\$(\\w+)");
     private static final Pattern HTTP_PARAM_PATTERN =
-            Pattern.compile("param (?<name>[\\w$]+)(?::(?<type>\\w+))?=(?<value>.*)", Pattern.MULTILINE);
+            Pattern.compile("param (?<name>[\\w-$]+)(?::(?<type>\\w+))?=(?<value>.*)", Pattern.MULTILINE);
     private static final Pattern HTTP_BODY_PATTERN = Pattern.compile("body=(.*)", Pattern.MULTILINE);
     private static final Pattern HTTP_STATUSCODE_PATTERN = Pattern.compile("statuscode (.*)", Pattern.MULTILINE);
     private static final Pattern MAX_RESULT_READS_PATTERN =
@@ -1493,12 +1502,14 @@
         while (m.find()) {
             final Parameter param = new Parameter();
             String name = m.group("name");
-            if (name.equals("parse")) {
+            param.setName(name);
+
+/*            if (name.equals("parse")) {
                 String newname = "parse-only";
                 param.setName(newname);
             } else {
                 param.setName(name);
-            }
+            }*/
             String value = m.group("value");
             param.setValue(value);
             String type = m.group("type");
diff --git a/asterixdb/asterix-app/src/test/resources/parseonly/queries_sqlpp/named_01.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/parseonly/queries_sqlpp/named_01.1.query.sqlpp
index a8c05a7..0a99965 100644
--- a/asterixdb/asterix-app/src/test/resources/parseonly/queries_sqlpp/named_01.1.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/parseonly/queries_sqlpp/named_01.1.query.sqlpp
@@ -23,7 +23,7 @@
  * Date         : Jun 2018
  */
 
--- param parse:string=true
+-- param parse-only:string=true
 
 // requesttype=application/json
 
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/named_01.1.parse.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/named_01.1.parse.sqlpp
index cf43e64..e24410f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/named_01.1.parse.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/named_01.1.parse.sqlpp
@@ -23,6 +23,6 @@
  * Date         : Jun 2018
  */
 
--- param parse:string=true
+-- param parse-only:string=true
 
 select count(*) from ChirpUsers where name=$qname;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/named_01.2.parse.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/named_01.2.parse.sqlpp
index a8c05a7..0a99965 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/named_01.2.parse.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/named_01.2.parse.sqlpp
@@ -23,7 +23,7 @@
  * Date         : Jun 2018
  */
 
--- param parse:string=true
+-- param parse-only:string=true
 
 // requesttype=application/json
 
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/named_01.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/named_01.1.adm
index 4d06619..66f16df 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/named_01.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/named_01.1.adm
@@ -1 +1 @@
-{"externalVariables":"[?qname,]"}
+{"statement-parameters":"[?qname,]"}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/named_01.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/named_01.2.adm
index decacc0..f38b229 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/named_01.2.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/named_01.2.adm
@@ -1 +1 @@
-{"externalVariables":"[?p_obj,?p_null,?p_bool,?p_dbl,?p_int,?p_str,?p_arr,?p_dec,]"}
+{"statement-parameters":"[?p_obj,?p_null,?p_bool,?p_dbl,?p_int,?p_str,?p_arr,?p_dec,]"}
diff --git a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
index 85f19ad..08c8d60 100644
--- a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
+++ b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
@@ -141,10 +141,16 @@
     }
 
     @Override
-    Set<VariableExpr> getFreeVariables(Expression expr) {
+    public Set<VariableExpr> getFreeVariables(Expression expr) {
         throw new UnsupportedOperationException("getFreeVariables not implemented for AQL");
     }
 
+    @Override
+    public Set<VariableExpr> getExternalVariables(Expression expr) {
+        throw new UnsupportedOperationException("getExternalVariables not implemented for AQL");
+    }
+
+
     private static class GatherFunctionCalls extends GatherFunctionCallsVisitor implements IAQLVisitor<Void, Void> {
 
         public GatherFunctionCalls() {
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java
index 0c42e40..852c17a 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java
@@ -53,7 +53,8 @@
      * Find the function calls used by a given expression
      */
     Set<CallExpr> getFunctionCalls(Expression expression) throws CompilationException;
-    Set<VariableExpr> getFreeVariables(Expression expr) throws CompilationException;
 
+    Set<VariableExpr> getFreeVariables(Expression expr) throws CompilationException;
+    Set<VariableExpr> getExternalVariables(Expression expr) throws CompilationException;
 
 }
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/ParseValueParser.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/ParseValueParser.java
deleted file mode 100644
index e765e83..0000000
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/ParseValueParser.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.lang.sqlpp.parser;
-
-import org.apache.asterix.common.exceptions.CompilationException;
-
-import org.apache.asterix.lang.common.base.IParser;
-import org.apache.asterix.lang.common.base.IParserFactory;
-import org.apache.asterix.lang.common.base.Statement;
-import org.apache.asterix.lang.common.expression.VariableExpr;
-import org.apache.asterix.lang.sqlpp.visitor.FreeVariableVisitor;
-import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppQueryExpressionVisitor;
-
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-public class ParseValueParser {
-    private final IParserFactory parserFactory;
-    private final ParseOnlyResult parseOnlyResult;
-    private final Set<VariableExpr> extVars;
-    private final String statementsText;
-    private IParser parser;
-
-    public ParseValueParser(IParserFactory parserFactory, String statementsText) {
-        this.statementsText = statementsText;
-        this.parserFactory = parserFactory;
-        this.extVars = new HashSet<>();
-        this.parseOnlyResult = new ParseOnlyResult();
-
-    }
-
-    public List<Statement> parse() throws CompilationException {
-        parser = parserFactory.createParser(statementsText);
-        List<Statement> statements = parser.parse();
-        AbstractSqlppQueryExpressionVisitor<Void, Collection<VariableExpr>> frv = new FreeVariableVisitor();
-        //NamedParameterVisitor erv = new NamedParameterVisitor();
-        for (Statement st : statements) {
-            st.accept(frv, extVars);
-        }
-        parseOnlyResult.setExternalVariables(extVars);
-        return statements;
-
-    }
-
-    public ParseOnlyResult getParseOnlyResult() {
-        return parseOnlyResult;
-    }
-
-}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
index 030c3bf..59daa88 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
@@ -66,6 +66,7 @@
 import org.apache.asterix.lang.sqlpp.rewrites.visitor.VariableCheckAndRewriteVisitor;
 import org.apache.asterix.lang.sqlpp.struct.SetOperationRight;
 import org.apache.asterix.lang.sqlpp.util.FunctionMapUtil;
+import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
 import org.apache.asterix.lang.sqlpp.visitor.FreeVariableVisitor;
 import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppQueryExpressionVisitor;
 import org.apache.asterix.lang.sqlpp.visitor.base.ISqlppVisitor;
@@ -252,11 +253,24 @@
     }
 
     @Override
-    public Set<VariableExpr> getFreeVariables(Expression expr) throws CompilationException
-    {
-        Set<VariableExpr> extVars = new HashSet<>(); //TODO: memory alloc within function?
+    public Set<VariableExpr> getFreeVariables(Expression expr) throws CompilationException {
+        Set<VariableExpr> freeVars = new HashSet<>(); //TODO: memory alloc within function?
         AbstractSqlppQueryExpressionVisitor<Void, Collection<VariableExpr>> frv = new FreeVariableVisitor();
-        expr.accept(frv, extVars);
+        expr.accept(frv, freeVars);
+        return freeVars;
+    }
+
+    @Override
+    public Set<VariableExpr> getExternalVariables(Expression expr) throws CompilationException {
+        Set<VariableExpr> freeVars = getFreeVariables(expr);
+        Set<VariableExpr> extVars = new HashSet<>();
+        for(VariableExpr ve : freeVars)
+        {
+            if(SqlppVariableUtil.isExternalVariableReference(ve))
+            {
+                extVars.add(ve);
+            }
+        }
         return extVars;
     }
 

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/3077
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I4b59edb9d4ad759dfe81c071d50f7eaa7000bdd1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Sandeep.gupta@couchbase.com