You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by "Ian Maxon (Code Review)" <do...@asterixdb.incubator.apache.org> on 2018/07/11 02:24:44 UTC

Change in asterixdb[release-0.9.4-pre-rc]: [NO ISSUE][API] Add Execution Plans to Query Services

Ian Maxon has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/2750

Change subject: [NO ISSUE][API] Add Execution Plans to Query Services
......................................................................

[NO ISSUE][API] Add Execution Plans to Query Services

- user model changes: no
- storage format changes: no
- interface changes: yes

- Add execution plans parameters to query service.
- Remove HTML code generation from APIFramework.

Change-Id: I99215243aae2cb96174671d109084a82af877335
---
M asterixdb/asterix-algebra/pom.xml
A asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlans.java
A asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansHtmlPrintUtil.java
A asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansJsonPrintUtil.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
M hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java
15 files changed, 398 insertions(+), 80 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/50/2750/1

diff --git a/asterixdb/asterix-algebra/pom.xml b/asterixdb/asterix-algebra/pom.xml
index 6a6bb8d..f3fdfc8 100644
--- a/asterixdb/asterix-algebra/pom.xml
+++ b/asterixdb/asterix-algebra/pom.xml
@@ -238,5 +238,9 @@
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hyracks</groupId>
+      <artifactId>hyracks-util</artifactId>
+    </dependency>
   </dependencies>
 </project>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlans.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlans.java
new file mode 100644
index 0000000..d77164c
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlans.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.translator;
+
+import java.io.Serializable;
+
+public class ExecutionPlans implements Serializable {
+
+    private String expressionTree;
+    private String rewrittenExpressionTree;
+    private String logicalPlan;
+    private String optimizedLogicalPlan;
+    private String job;
+
+    public String getExpressionTree() {
+        return expressionTree;
+    }
+
+    public void setExpressionTree(String expressionTree) {
+        this.expressionTree = expressionTree;
+    }
+
+    public String getRewrittenExpressionTree() {
+        return rewrittenExpressionTree;
+    }
+
+    public void setRewrittenExpressionTree(String rewrittenExpressionTree) {
+        this.rewrittenExpressionTree = rewrittenExpressionTree;
+    }
+
+    public String getLogicalPlan() {
+        return logicalPlan;
+    }
+
+    public void setLogicalPlan(String logicalPlan) {
+        this.logicalPlan = logicalPlan;
+    }
+
+    public String getOptimizedLogicalPlan() {
+        return optimizedLogicalPlan;
+    }
+
+    public void setOptimizedLogicalPlan(String optimizedLogicalPlan) {
+        this.optimizedLogicalPlan = optimizedLogicalPlan;
+    }
+
+    public String getJob() {
+        return job;
+    }
+
+    public void setJob(String job) {
+        this.job = job;
+    }
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansHtmlPrintUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansHtmlPrintUtil.java
new file mode 100644
index 0000000..88e8255
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansHtmlPrintUtil.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.translator;
+
+import java.io.PrintWriter;
+
+public class ExecutionPlansHtmlPrintUtil {
+
+    private static final String LOGICAL_PLAN_LBL = "Logical plan";
+    private static final String EXPRESSION_TREE_LBL = "Expression tree";
+    private static final String REWRITTEN_EXPRESSION_TREE_LBL = "Rewritten expression tree";
+    private static final String OPTIMIZED_LOGICAL_PLAN_LBL = "Optimized logical plan";
+    private static final String JOB_LBL = "Job";
+
+    private ExecutionPlansHtmlPrintUtil() {
+    }
+
+    public static void print(PrintWriter output, ExecutionPlans plans) {
+        printNonNull(output, EXPRESSION_TREE_LBL, plans.getExpressionTree());
+        printNonNull(output, REWRITTEN_EXPRESSION_TREE_LBL, plans.getRewrittenExpressionTree());
+        printNonNull(output, LOGICAL_PLAN_LBL, plans.getLogicalPlan());
+        printNonNull(output, OPTIMIZED_LOGICAL_PLAN_LBL, plans.getOptimizedLogicalPlan());
+        printNonNull(output, JOB_LBL, plans.getJob());
+    }
+
+    private static void printNonNull(PrintWriter output, String lbl, String value) {
+        if (value != null) {
+            printFieldPrefix(output, lbl);
+            output.print(value);
+            printFieldPostfix(output);
+        }
+    }
+
+    private static void printFieldPrefix(PrintWriter output, String lbl) {
+        output.println();
+        output.println("<h4>" + lbl + ":</h4>");
+        switch (lbl) {
+            case LOGICAL_PLAN_LBL:
+                output.println("<pre class=query-plan>");
+                break;
+            case OPTIMIZED_LOGICAL_PLAN_LBL:
+                output.println("<pre class=query-optimized-plan>");
+                break;
+            default:
+                output.println("<pre>");
+                break;
+        }
+    }
+
+    private static void printFieldPostfix(PrintWriter output) {
+        output.println("</pre>");
+    }
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansJsonPrintUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansJsonPrintUtil.java
new file mode 100644
index 0000000..5c47ca2
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ExecutionPlansJsonPrintUtil.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.translator;
+
+import static org.apache.asterix.translator.SessionConfig.PlanFormat.STRING;
+
+import org.apache.hyracks.util.JSONUtil;
+
+public class ExecutionPlansJsonPrintUtil {
+
+    private static final String LOGICAL_PLAN_LBL = "logicalPlan";
+    private static final String EXPRESSION_TREE_LBL = "expressionTree";
+    private static final String REWRITTEN_EXPRESSION_TREE_LBL = "rewrittenExpressionTree";
+    private static final String OPTIMIZED_LOGICAL_PLAN_LBL = "optimizedLogicalPlan";
+    private static final String JOB_LBL = "job";
+
+    private ExecutionPlansJsonPrintUtil() {
+    }
+
+    public static String asJson(ExecutionPlans plans, SessionConfig.PlanFormat format) {
+        final StringBuilder output = new StringBuilder();
+        appendOutputPrefix(output);
+        // TODO only string is currently supported for expression trees
+        appendNonNull(output, EXPRESSION_TREE_LBL, plans.getExpressionTree(), STRING);
+        appendNonNull(output, REWRITTEN_EXPRESSION_TREE_LBL, plans.getRewrittenExpressionTree(), STRING);
+        appendNonNull(output, LOGICAL_PLAN_LBL, plans.getLogicalPlan(), format);
+        appendNonNull(output, OPTIMIZED_LOGICAL_PLAN_LBL, plans.getOptimizedLogicalPlan(), format);
+        appendNonNull(output, JOB_LBL, plans.getJob(), format);
+        appendOutputPostfix(output);
+        return output.toString();
+    }
+
+    private static void appendNonNull(StringBuilder builder, String lbl, String value,
+            SessionConfig.PlanFormat format) {
+        if (value != null) {
+            printFieldPrefix(builder, lbl);
+            switch (format) {
+                case JSON:
+                    builder.append(value);
+                    break;
+                case STRING:
+                    JSONUtil.quoteAndEscape(builder, value);
+                    break;
+                default:
+                    throw new IllegalStateException("Unrecognized plan format: " + format);
+            }
+            printFieldPostfix(builder);
+        }
+    }
+
+    private static void appendOutputPrefix(StringBuilder builder) {
+        builder.append("{");
+    }
+
+    private static void printFieldPrefix(StringBuilder builder, String lbl) {
+        builder.append("\"" + lbl + "\": ");
+    }
+
+    private static void printFieldPostfix(StringBuilder builder) {
+        builder.append(",");
+    }
+
+    private static void appendOutputPostfix(StringBuilder builder) {
+        // remove extra comma if needed
+        if (builder.length() > 1) {
+            builder.deleteCharAt(builder.length() - 1);
+        }
+        builder.append("}");
+    }
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
index d76c421..0ff877b 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
@@ -142,4 +142,11 @@
      */
     String getActiveDataverseName(String dataverse);
 
+    /**
+     * Gets the execution plans that are generated during query compilation
+     *
+     * @return the executions plans
+     */
+    ExecutionPlans getExecutionPlans();
+
 }
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
index cb6d8e5..89619e5 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
@@ -63,7 +63,7 @@
                 if (fmtString != null) {
                     String format = ("JSON".equalsIgnoreCase(fmtString) || "CLEAN_JSON".equalsIgnoreCase(fmtString))
                             ? "JSON" : fmtString;
-                    return PlanFormat.valueOf(format);
+                    return PlanFormat.valueOf(format.toUpperCase());
                 }
             } catch (IllegalArgumentException e) {
                 logger.log(Level.INFO, fmtString + ": unsupported " + label + ", using " + defaultFmt + "instead", e);
@@ -129,7 +129,7 @@
 
     // Output format.
     private final OutputFormat fmt;
-    private final PlanFormat lpfmt;
+    private final PlanFormat planFormat;
 
     // Standard execution flags.
     private final boolean executeQuery;
@@ -143,8 +143,8 @@
         this(fmt, PlanFormat.STRING);
     }
 
-    public SessionConfig(OutputFormat fmt, PlanFormat lpfmt) {
-        this(fmt, true, true, true, lpfmt);
+    public SessionConfig(OutputFormat fmt, PlanFormat planFormat) {
+        this(fmt, true, true, true, planFormat);
     }
 
     /**
@@ -168,13 +168,13 @@
     }
 
     public SessionConfig(OutputFormat fmt, boolean optimize, boolean executeQuery, boolean generateJobSpec,
-            PlanFormat lpfmt) {
+            PlanFormat planFormat) {
         this.fmt = fmt;
         this.optimize = optimize;
         this.executeQuery = executeQuery;
         this.generateJobSpec = generateJobSpec;
         this.flags = new HashMap<>();
-        this.lpfmt = lpfmt;
+        this.planFormat = planFormat;
     }
 
     /**
@@ -187,8 +187,8 @@
     /**
      * Retrieve the PlanFormat for this execution.
      */
-    public PlanFormat getLpfmt() {
-        return this.lpfmt;
+    public PlanFormat getPlanFormat() {
+        return this.planFormat;
     }
 
     /**
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index ad715a4..537625d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -67,6 +68,7 @@
 import org.apache.asterix.optimizer.rules.am.AbstractIntroduceAccessMethodRule;
 import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
 import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
+import org.apache.asterix.translator.ExecutionPlans;
 import org.apache.asterix.translator.IStatementExecutor.Stats;
 import org.apache.asterix.translator.SessionConfig;
 import org.apache.asterix.translator.SessionOutput;
@@ -120,8 +122,7 @@
     private static final int MIN_FRAME_LIMIT_FOR_JOIN = 5;
     // one for query, two for intermediate results, one for final result, and one for reading an inverted list
     private static final int MIN_FRAME_LIMIT_FOR_TEXTSEARCH = 5;
-    private static final String LPLAN = "Logical plan";
-    private static final String OPLAN = "Optimized logical plan";
+    private static final ObjectWriter OBJECT_WRITER = new ObjectMapper().writerWithDefaultPrettyPrinter();
 
     // A white list of supported configurable parameters.
     private static final Set<String> CONFIGURABLE_PARAMETER_NAMES =
@@ -137,12 +138,14 @@
     private final IAstPrintVisitorFactory astPrintVisitorFactory;
     private final ILangExpressionToPlanTranslatorFactory translatorFactory;
     private final IRuleSetFactory ruleSetFactory;
+    private final ExecutionPlans executionPlans;
 
     public APIFramework(ILangCompilationProvider compilationProvider) {
         this.rewriterFactory = compilationProvider.getRewriterFactory();
         this.astPrintVisitorFactory = compilationProvider.getAstPrintVisitorFactory();
         this.translatorFactory = compilationProvider.getExpressionToPlanTranslatorFactory();
         this.ruleSetFactory = compilationProvider.getRuleSetFactory();
+        executionPlans = new ExecutionPlans();
     }
 
     private static class OptimizationContextFactory implements IOptimizationContextFactory {
@@ -165,27 +168,6 @@
         }
     }
 
-    private void printPlanPrefix(SessionOutput output, String planName) {
-        if (output.config().is(SessionConfig.FORMAT_HTML)) {
-            output.out().println("<h4>" + planName + ":</h4>");
-            if (LPLAN.equalsIgnoreCase(planName)) {
-                output.out().println("<pre class = query-plan>");
-            } else if (OPLAN.equalsIgnoreCase(planName)) {
-                output.out().println("<pre class = query-optimized-plan>");
-            } else {
-                output.out().println("<pre>");
-            }
-        } else {
-            output.out().println("----------" + planName + ":");
-        }
-    }
-
-    private void printPlanPostfix(SessionOutput output) {
-        if (output.config().is(SessionConfig.FORMAT_HTML)) {
-            output.out().println("</pre>");
-        }
-    }
-
     public Pair<IReturningStatement, Integer> reWriteQuery(List<FunctionDecl> declaredFunctions,
             MetadataProvider metadataProvider, IReturningStatement q, SessionOutput output, boolean inlineUdfs)
             throws CompilationException {
@@ -194,10 +176,7 @@
         }
         SessionConfig conf = output.config();
         if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_EXPR_TREE)) {
-            output.out().println();
-            printPlanPrefix(output, "Expression tree");
-            q.accept(astPrintVisitorFactory.createLangVisitor(output.out()), 0);
-            printPlanPostfix(output);
+            generateExpressionTree(q);
         }
         IQueryRewriter rw = rewriterFactory.createQueryRewriter();
         rw.rewrite(declaredFunctions, q, metadataProvider, new LangRewritingContext(q.getVarCounter()), inlineUdfs);
@@ -213,14 +192,9 @@
         final boolean isLoad = statement != null && statement.getKind() == Statement.Kind.LOAD;
 
         SessionConfig conf = output.config();
-        if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_REWRITTEN_EXPR_TREE)) {
-            output.out().println();
-
-            printPlanPrefix(output, "Rewritten expression tree");
-            if (isQuery) {
-                query.accept(astPrintVisitorFactory.createLangVisitor(output.out()), 0);
-            }
-            printPlanPostfix(output);
+        if (isQuery && !conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS)
+                && conf.is(SessionConfig.OOB_REWRITTEN_EXPR_TREE)) {
+            generateRewrittenExpressionTree(query);
         }
 
         final TxnId txnId = metadataProvider.getTxnIdFactory().create();
@@ -230,14 +204,9 @@
 
         ILogicalPlan plan = isLoad ? t.translateLoad(statement) : t.translate(query, outputDatasetName, statement);
 
-        if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_LOGICAL_PLAN)) {
-            output.out().println();
-
-            printPlanPrefix(output, "Logical plan");
-            if (isQuery || isLoad) {
-                PlanPrettyPrinter.printPlan(plan, getPrettyPrintVisitor(output.config().getLpfmt(), output.out()), 0);
-            }
-            printPlanPostfix(output);
+        if ((isQuery || isLoad) && !conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS)
+                && conf.is(SessionConfig.OOB_LOGICAL_PLAN)) {
+            generateLogicalPlan(plan, output.config().getPlanFormat());
         }
         CompilerProperties compilerProperties = metadataProvider.getApplicationContext().getCompilerProperties();
         Map<String, String> querySpecificConfig = validateConfig(metadataProvider.getConfig());
@@ -273,12 +242,9 @@
                     AlgebricksAppendable buffer = new AlgebricksAppendable(output.out());
                     PlanPrettyPrinter.printPhysicalOps(plan, buffer, 0);
                 } else {
-                    printPlanPrefix(output, "Optimized logical plan");
                     if (isQuery || isLoad) {
-                        PlanPrettyPrinter.printPlan(plan,
-                                getPrettyPrintVisitor(output.config().getLpfmt(), output.out()), 0);
+                        generateOptimizedLogicalPlan(plan, output.config().getPlanFormat());
                     }
-                    printPlanPostfix(output);
                 }
             }
         }
@@ -327,8 +293,9 @@
                     ResourceUtils.getRequiredCapacity(plan, jobLocations, physOptConf);
             spec.setRequiredClusterCapacity(jobRequiredCapacity);
         }
-
-        printJobSpec(query, spec, conf, output);
+        if (isQuery && conf.is(SessionConfig.OOB_HYRACKS_JOB)) {
+            generateJob(spec);
+        }
         return spec;
     }
 
@@ -373,23 +340,6 @@
         }
     }
 
-    protected void printJobSpec(Query rwQ, JobSpecification spec, SessionConfig conf, SessionOutput output)
-            throws AlgebricksException {
-        if (conf.is(SessionConfig.OOB_HYRACKS_JOB)) {
-            printPlanPrefix(output, "Hyracks job");
-            if (rwQ != null) {
-                try {
-                    final ObjectWriter objectWriter = new ObjectMapper().writerWithDefaultPrettyPrinter();
-                    output.out().println(objectWriter.writeValueAsString(spec.toJSON()));
-                } catch (IOException e) {
-                    throw new AlgebricksException(e);
-                }
-                output.out().println(spec.getUserConstraints());
-            }
-            printPlanPostfix(output);
-        }
-    }
-
     private AbstractLogicalOperatorPrettyPrintVisitor getPrettyPrintVisitor(SessionConfig.PlanFormat planFormat,
             PrintWriter out) {
         return planFormat.equals(SessionConfig.PlanFormat.JSON) ? new LogicalOperatorPrettyPrintVisitorJson(out)
@@ -427,6 +377,10 @@
             double duration = (endTime - startTime) / 1000.00;
             out.println("<pre>Duration: " + duration + " sec</pre>");
         }
+    }
+
+    public ExecutionPlans getExecutionPlans() {
+        return executionPlans;
     }
 
     // Chooses the location constraints, i.e., whether to use storage parallelism or use a user-sepcified number
@@ -524,6 +478,49 @@
         return config;
     }
 
+    private void generateExpressionTree(IReturningStatement statement) throws CompilationException {
+        final StringWriter stringWriter = new StringWriter();
+        try (PrintWriter writer = new PrintWriter(stringWriter)) {
+            statement.accept(astPrintVisitorFactory.createLangVisitor(writer), 0);
+            executionPlans.setExpressionTree(stringWriter.toString());
+        }
+    }
+
+    private void generateRewrittenExpressionTree(IReturningStatement statement) throws CompilationException {
+        final StringWriter stringWriter = new StringWriter();
+        try (PrintWriter writer = new PrintWriter(stringWriter)) {
+            statement.accept(astPrintVisitorFactory.createLangVisitor(writer), 0);
+            executionPlans.setRewrittenExpressionTree(stringWriter.toString());
+        }
+    }
+
+    private void generateLogicalPlan(ILogicalPlan plan, SessionConfig.PlanFormat format) throws AlgebricksException {
+        final StringWriter stringWriter = new StringWriter();
+        try (PrintWriter writer = new PrintWriter(stringWriter)) {
+            PlanPrettyPrinter.printPlan(plan, getPrettyPrintVisitor(format, writer), 0);
+            executionPlans.setLogicalPlan(stringWriter.toString());
+        }
+    }
+
+    private void generateOptimizedLogicalPlan(ILogicalPlan plan, SessionConfig.PlanFormat format)
+            throws AlgebricksException {
+        final StringWriter stringWriter = new StringWriter();
+        try (PrintWriter writer = new PrintWriter(stringWriter)) {
+            PlanPrettyPrinter.printPlan(plan, getPrettyPrintVisitor(format, writer), 0);
+            executionPlans.setOptimizedLogicalPlan(stringWriter.toString());
+        }
+    }
+
+    private void generateJob(JobSpecification spec) {
+        final StringWriter stringWriter = new StringWriter();
+        try (PrintWriter writer = new PrintWriter(stringWriter)) {
+            writer.println(OBJECT_WRITER.writeValueAsString(spec.toJSON()));
+            executionPlans.setJob(stringWriter.toString());
+        } catch (IOException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+
     public static AlgebricksAbsolutePartitionConstraint getJobLocations(JobSpecification spec,
             INodeJobTracker jobTracker, AlgebricksAbsolutePartitionConstraint clusterLocations) {
         final Set<String> jobParticipatingNodes = jobTracker.getJobParticipatingNodes(spec);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
index b8c737d..bd096dd 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
@@ -51,7 +51,8 @@
         RESULTS("results"),
         HANDLE("handle"),
         ERRORS("errors"),
-        METRICS("metrics");
+        METRICS("metrics"),
+        PLANS("plans");
 
         private final String str;
 
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
index a420efc..1713ca5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
@@ -132,6 +132,7 @@
         } else {
             sessionOutput.out().append(responseMsg.getResult());
         }
+        printExecutionPlans(sessionOutput, responseMsg.getExecutionPlans());
     }
 
     private void cancelQuery(INCMessageBroker messageBroker, String nodeId, String clientContextID, Exception exception,
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index 56359e3..714bb53 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -45,6 +45,8 @@
 import org.apache.asterix.lang.common.base.IParser;
 import org.apache.asterix.lang.common.base.Statement;
 import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.translator.ExecutionPlans;
+import org.apache.asterix.translator.ExecutionPlansJsonPrintUtil;
 import org.apache.asterix.translator.IRequestParameters;
 import org.apache.asterix.translator.IStatementExecutor;
 import org.apache.asterix.translator.IStatementExecutor.ResultDelivery;
@@ -73,6 +75,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+
 import io.netty.handler.codec.http.HttpResponseStatus;
 
 public class QueryServiceServlet extends AbstractQueryApiServlet {
@@ -141,7 +144,12 @@
         MODE("mode"),
         TIMEOUT("timeout"),
         PLAN_FORMAT("plan-format"),
-        MAX_RESULT_READS("max-result-reads");
+        MAX_RESULT_READS("max-result-reads"),
+        EXPRESSION_TREE("expression-tree"),
+        REWRITTEN_EXPRESSION_TREE("rewritten-expression-tree"),
+        LOGICAL_PLAN("logical-plan"),
+        OPTIMIZED_LOGICAL_PLAN("optimized-logical-plan"),
+        JOB("job");
 
         private final String str;
 
@@ -198,6 +206,12 @@
         String clientContextID;
         String mode;
         String maxResultReads;
+        String planFormat;
+        boolean expressionTree;
+        boolean rewrittenExpressionTree;
+        boolean logicalPlan;
+        boolean optimizedLogicalPlan;
+        boolean job;
 
         @Override
         public String toString() {
@@ -213,6 +227,12 @@
                 on.put("format", format);
                 on.put("timeout", timeout);
                 on.put("maxResultReads", maxResultReads);
+                on.put("planFormat", planFormat);
+                on.put("expressionTree", expressionTree);
+                on.put("rewrittenExpressionTree", rewrittenExpressionTree);
+                on.put("logicalPlan", logicalPlan);
+                on.put("optimizedLogicalPlan", optimizedLogicalPlan);
+                on.put("job", job);
                 return om.writer(new MinimalPrettyPrinter()).writeValueAsString(on);
             } catch (JsonProcessingException e) { // NOSONAR
                 LOGGER.debug("unexpected exception marshalling {} instance to json", getClass(), e);
@@ -307,9 +327,15 @@
         SessionOutput.ResultAppender appendStatus = ResultUtil.createResultStatusAppender();
 
         SessionConfig.OutputFormat format = getFormat(param.format);
-        //TODO:get the parameters from UI.Currently set to clean_json.
-        SessionConfig sessionConfig = new SessionConfig(format);
+        final SessionConfig.PlanFormat planFormat =
+                SessionConfig.PlanFormat.get(param.planFormat, param.planFormat, SessionConfig.PlanFormat.JSON, LOGGER);
+        SessionConfig sessionConfig = new SessionConfig(format, planFormat);
         sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, true);
+        sessionConfig.set(SessionConfig.OOB_EXPR_TREE, param.expressionTree);
+        sessionConfig.set(SessionConfig.OOB_REWRITTEN_EXPR_TREE, param.rewrittenExpressionTree);
+        sessionConfig.set(SessionConfig.OOB_LOGICAL_PLAN, param.logicalPlan);
+        sessionConfig.set(SessionConfig.OOB_OPTIMIZED_LOGICAL_PLAN, param.optimizedLogicalPlan);
+        sessionConfig.set(SessionConfig.OOB_HYRACKS_JOB, param.job);
         sessionConfig.set(SessionConfig.FORMAT_INDENT_JSON, param.pretty);
         sessionConfig.set(SessionConfig.FORMAT_QUOTE_RECORD,
                 format != SessionConfig.OutputFormat.CLEAN_JSON && format != SessionConfig.OutputFormat.LOSSLESS_JSON);
@@ -391,6 +417,13 @@
                 param.clientContextID = getOptText(jsonRequest, Parameter.CLIENT_ID.str());
                 param.timeout = getOptText(jsonRequest, Parameter.TIMEOUT.str());
                 param.maxResultReads = getOptText(jsonRequest, Parameter.MAX_RESULT_READS.str());
+                param.planFormat = getOptText(jsonRequest, Parameter.PLAN_FORMAT.str());
+                param.expressionTree = getOptBoolean(jsonRequest, Parameter.EXPRESSION_TREE.str(), false);
+                param.rewrittenExpressionTree =
+                        getOptBoolean(jsonRequest, Parameter.REWRITTEN_EXPRESSION_TREE.str(), false);
+                param.logicalPlan = getOptBoolean(jsonRequest, Parameter.LOGICAL_PLAN.str(), false);
+                param.optimizedLogicalPlan = getOptBoolean(jsonRequest, Parameter.OPTIMIZED_LOGICAL_PLAN.str(), false);
+                param.job = getOptBoolean(jsonRequest, Parameter.JOB.str(), false);
             } catch (JsonParseException | JsonMappingException e) {
                 // if the JSON parsing fails, the statement is empty and we get an empty statement error
                 GlobalConfig.ASTERIX_LOGGER.log(Level.ERROR, e.getMessage(), e);
@@ -406,6 +439,7 @@
             param.clientContextID = request.getParameter(Parameter.CLIENT_ID.str());
             param.timeout = request.getParameter(Parameter.TIMEOUT.str());
             param.maxResultReads = request.getParameter(Parameter.MAX_RESULT_READS.str());
+            param.planFormat = request.getParameter(Parameter.PLAN_FORMAT.str());
         }
         return param;
     }
@@ -533,6 +567,7 @@
                 getHyracksDataset(), resultProperties, stats, null, param.clientContextID, optionalParameters);
         translator.compileAndExecute(getHyracksClientConnection(), queryCtx, requestParameters);
         execution.end();
+        printExecutionPlans(sessionOutput, translator.getExecutionPlans());
     }
 
     protected void handleExecuteStatementException(Throwable t, RequestExecutionState state, RequestParameters param) {
@@ -566,4 +601,21 @@
             state.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR);
         }
     }
+
+    protected void printExecutionPlans(SessionOutput output, ExecutionPlans executionPlans) {
+        final PrintWriter pw = output.out();
+        pw.print("\t\"");
+        pw.print(ResultFields.PLANS.str());
+        pw.print("\":");
+        final SessionConfig.PlanFormat planFormat = output.config().getPlanFormat();
+        switch (planFormat) {
+            case JSON:
+            case STRING:
+                pw.print(ExecutionPlansJsonPrintUtil.asJson(executionPlans, planFormat));
+                break;
+            default:
+                throw new IllegalStateException("Unrecognized plan format: " + planFormat);
+        }
+        pw.print(",\n");
+    }
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
index 53d4f3f..d295304 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
@@ -131,6 +131,7 @@
             responseMsg.setResult(outWriter.toString());
             responseMsg.setMetadata(outMetadata);
             responseMsg.setStats(stats);
+            responseMsg.setExecutionPlans(translator.getExecutionPlans());
         } catch (AlgebricksException | HyracksException | TokenMgrError
                 | org.apache.asterix.aqlplus.parser.TokenMgrError pe) {
             // we trust that "our" exceptions are serializable and have a comprehensible error message
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java
index 7475be4..94dd541 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java
@@ -23,6 +23,7 @@
 import org.apache.asterix.common.messaging.api.INcAddressedMessage;
 import org.apache.asterix.common.messaging.api.MessageFuture;
 import org.apache.asterix.messaging.NCMessageBroker;
+import org.apache.asterix.translator.ExecutionPlans;
 import org.apache.asterix.translator.IStatementExecutor;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 
@@ -38,6 +39,8 @@
     private IStatementExecutor.Stats stats;
 
     private Throwable error;
+
+    private ExecutionPlans executionPlans;
 
     public ExecuteStatementResponseMessage(long requestMessageId) {
         this.requestMessageId = requestMessageId;
@@ -84,6 +87,14 @@
         this.stats = stats;
     }
 
+    public ExecutionPlans getExecutionPlans() {
+        return executionPlans;
+    }
+
+    public void setExecutionPlans(ExecutionPlans executionPlans) {
+        this.executionPlans = executionPlans;
+    }
+
     @Override
     public String toString() {
         return String.format("%s(id=%s): %d characters", getClass().getSimpleName(), requestMessageId,
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index e983763..3fd05e2 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -156,6 +156,8 @@
 import org.apache.asterix.translator.CompiledStatements.CompiledLoadFromFileStatement;
 import org.apache.asterix.translator.CompiledStatements.CompiledUpsertStatement;
 import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
+import org.apache.asterix.translator.ExecutionPlans;
+import org.apache.asterix.translator.ExecutionPlansHtmlPrintUtil;
 import org.apache.asterix.translator.IRequestParameters;
 import org.apache.asterix.translator.IStatementExecutor;
 import org.apache.asterix.translator.IStatementExecutorContext;
@@ -1775,6 +1777,7 @@
                     new CompiledLoadFromFileStatement(dataverseName, loadStmt.getDatasetName().getValue(),
                             loadStmt.getAdapter(), loadStmt.getProperties(), loadStmt.dataIsAlreadySorted());
             JobSpecification spec = apiFramework.compileQuery(hcc, metadataProvider, null, 0, null, sessionOutput, cls);
+            afterCompile();
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
             bActiveTxn = false;
             if (spec != null) {
@@ -1866,6 +1869,7 @@
                     stmtDelete.getDatasetName().getValue(), stmtDelete.getCondition(), stmtDelete.getVarCounter(),
                     stmtDelete.getQuery());
             JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, clfrqs.getQuery(), clfrqs);
+            afterCompile();
 
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
             bActiveTxn = false;
@@ -2372,6 +2376,7 @@
             metadataProvider.setMetadataTxnContext(mdTxnCtx);
             try {
                 final JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, query, null);
+                afterCompile();
                 MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
                 bActiveTxn = false;
                 return query.isExplain() || !sessionConfig.isExecuteQuery() ? null : jobSpec;
@@ -2780,6 +2785,11 @@
         return (dataverse != null) ? dataverse : activeDataverse.getDataverseName();
     }
 
+    @Override
+    public ExecutionPlans getExecutionPlans() {
+        return apiFramework.getExecutionPlans();
+    }
+
     public String getActiveDataverse(Identifier dataverse) {
         return getActiveDataverseName(dataverse != null ? dataverse.getValue() : null);
     }
@@ -2819,5 +2829,12 @@
         if (index.isPrimaryIndex()) {
             throw new MetadataException(ErrorCode.CANNOT_DROP_INDEX, index.getIndexName(), index.getDatasetName());
         }
+
+    }
+
+    protected void afterCompile() {
+        if (sessionOutput.config().is(SessionConfig.FORMAT_HTML)) {
+            ExecutionPlansHtmlPrintUtil.print(sessionOutput.out(), getExecutionPlans());
+        }
     }
 }
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
index a356d23..43833a2 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
@@ -53,7 +53,8 @@
         SIGNATURE("signature"),
         STATUS("status"),
         TYPE("type"),
-        ERRORS("errors");
+        ERRORS("errors"),
+        PLANS("plans");
 
         private static final Map<String, ResultField> fields = new HashMap<>();
 
@@ -162,6 +163,7 @@
                 case SIGNATURE:
                 case STATUS:
                 case TYPE:
+                case PLANS:
                     resultBuilder.append(OBJECT_MAPPER.writeValueAsString(fieldValue));
                     break;
                 default:
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java
index 6085c1c..baa3174 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java
@@ -149,7 +149,7 @@
         return quoteAndEscape(new StringBuilder(), str).toString();
     }
 
-    private static StringBuilder quoteAndEscape(StringBuilder sb, String str) {
+    public static StringBuilder quoteAndEscape(StringBuilder sb, String str) {
         return escape(sb.append('"'), str).append('"');
     }
 

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/2750
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I99215243aae2cb96174671d109084a82af877335
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: release-0.9.4-pre-rc
Gerrit-Owner: Ian Maxon <im...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <mh...@apache.org>