You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ce...@apache.org on 2016/02/06 02:22:52 UTC
incubator-asterixdb git commit: Add QueryServiceServlet
Repository: incubator-asterixdb
Updated Branches:
refs/heads/master d434b50f8 -> fa8a7ade3
Add QueryServiceServlet
Adds a new improved HTTP endpoint for queries. Also introduces
initial stats gathering.
Change-Id: Ia494c54f7252445ce38903c0b58fc4e23c324e6e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/597
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Chris Hillery <ce...@lambda.nu>
Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/fa8a7ade
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/fa8a7ade
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/fa8a7ade
Branch: refs/heads/master
Commit: fa8a7ade35857a97e1444a5e27e199c3ad4923c8
Parents: d434b50
Author: Till Westmann <ti...@apache.org>
Authored: Mon Jan 25 16:09:00 2016 -0800
Committer: Chris Hillery <ce...@lambda.nu>
Committed: Fri Feb 5 10:30:54 2016 -0800
----------------------------------------------------------------------
.../common/AsterixHyracksIntegrationUtil.java | 1 +
.../asterix/api/common/SessionConfig.java | 91 +++--
.../asterix/api/http/servlet/JSONUtil.java | 132 +++++++
.../api/http/servlet/QueryResultAPIServlet.java | 7 +-
.../api/http/servlet/QueryServiceServlet.java | 342 +++++++++++++++++++
.../asterix/aql/translator/QueryTranslator.java | 15 +-
.../bootstrap/CCApplicationEntryPoint.java | 5 +-
.../org/apache/asterix/result/ResultUtils.java | 15 +-
.../metadata/bootstrap/MetadataBootstrap.java | 2 +-
9 files changed, 573 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fa8a7ade/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
index d815f30..e33aed2 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -190,6 +190,7 @@ public class AsterixHyracksIntegrationUtil {
}
} catch (Exception e) {
e.printStackTrace();
+ System.exit(1);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fa8a7ade/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java b/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java
index f0a0cc2..2ed67b2 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java
@@ -25,15 +25,15 @@ import java.util.Map;
/**
* SessionConfig captures several different parameters for controlling
* the execution of an APIFramework call.
- * <li> It specifies how the execution will proceed (for instance,
+ * <li>It specifies how the execution will proceed (for instance,
* whether to optimize, or whether to execute at all).
- * <li> It allows you specify where the primary execution output will
+ * <li>It allows you specify where the primary execution output will
* be sent.
- * <li> It also allows you to request additional output for optional
+ * <li>It also allows you to request additional output for optional
* out-of-band data about the execution (query plan, etc).
- * <li> It allows you to specify the output format for the primary
+ * <li>It allows you to specify the output format for the primary
* execution output - LOSSLESS_JSON, CSV, etc.
- * <li> It allows you to specify output format-specific parameters.
+ * <li>It allows you to specify output format-specific parameters.
*/
public class SessionConfig {
@@ -92,6 +92,10 @@ public class SessionConfig {
*/
public static final String FORMAT_WRAPPER_ARRAY = "format-wrapper-array";
+ public interface ResultDecorator {
+ PrintWriter print(PrintWriter pw);
+ }
+
// Standard execution flags.
private final boolean executeQuery;
private final boolean generateJobSpec;
@@ -103,43 +107,66 @@ public class SessionConfig {
// Output format.
private final OutputFormat fmt;
+ private final ResultDecorator preResultDecorator;
+ private final ResultDecorator postResultDecorator;
+
// Flags.
- private final Map<String,Boolean> flags;
+ private final Map<String, Boolean> flags;
/**
* Create a SessionConfig object with all default values:
- *
* - All format flags set to "false".
* - All out-of-band outputs set to "null".
* - "Optimize" set to "true".
* - "Execute Query" set to "true".
* - "Generate Job Spec" set to "true".
- * @param out PrintWriter for execution output.
- * @param fmt Output format for execution output.
+ *
+ * @param out
+ * PrintWriter for execution output.
+ * @param fmt
+ * Output format for execution output.
*/
public SessionConfig(PrintWriter out, OutputFormat fmt) {
- this(out, fmt, true, true, true);
+ this(out, fmt, null, null, true, true, true);
+ }
+
+ public SessionConfig(PrintWriter out, OutputFormat fmt, ResultDecorator preResultDecorator,
+ ResultDecorator postResultDecorator) {
+ this(out, fmt, preResultDecorator, postResultDecorator, true, true, true);
+ }
+
+ public SessionConfig(PrintWriter out, OutputFormat fmt, boolean optimize, boolean executeQuery,
+ boolean generateJobSpec) {
+ this(out, fmt, null, null, optimize, executeQuery, generateJobSpec);
}
/**
* Create a SessionConfig object with all optional values set to defaults:
- *
* - All format flags set to "false".
* - All out-of-band outputs set to "false".
- * @param out PrintWriter for execution output.
- * @param fmt Output format for execution output.
- * @param optimize Whether to optimize the execution.
- * @param executeQuery Whether to execute the query or not.
- * @param generateJobSpec Whether to generate the Hyracks job specification (if
- * false, job cannot be executed).
- */
- public SessionConfig(PrintWriter out, OutputFormat fmt, boolean optimize, boolean executeQuery, boolean generateJobSpec) {
+ *
+ * @param out
+ * PrintWriter for execution output.
+ * @param fmt
+ * Output format for execution output.
+ * @param optimize
+ * Whether to optimize the execution.
+ * @param executeQuery
+ * Whether to execute the query or not.
+ * @param generateJobSpec
+ * Whether to generate the Hyracks job specification (if
+ * false, job cannot be executed).
+ */
+ public SessionConfig(PrintWriter out, OutputFormat fmt, ResultDecorator preResultDecorator,
+ ResultDecorator postResultDecorator, boolean optimize, boolean executeQuery, boolean generateJobSpec) {
this.out = out;
this.fmt = fmt;
+ this.preResultDecorator = preResultDecorator;
+ this.postResultDecorator = postResultDecorator;
this.optimize = optimize;
this.executeQuery = executeQuery;
this.generateJobSpec = generateJobSpec;
- this.flags = new HashMap<String,Boolean>();
+ this.flags = new HashMap<String, Boolean>();
}
/**
@@ -156,6 +183,14 @@ public class SessionConfig {
return this.fmt;
}
+ public PrintWriter resultPrefix(PrintWriter pw) {
+ return this.preResultDecorator != null ? this.preResultDecorator.print(pw) : pw;
+ };
+
+ public PrintWriter resultPostfix(PrintWriter pw) {
+ return this.postResultDecorator != null ? this.postResultDecorator.print(pw) : pw;
+ };
+
/**
* Retrieve the value of the "execute query" flag.
*/
@@ -180,9 +215,8 @@ public class SessionConfig {
/**
* Specify all out-of-band settings at once. For convenience of older code.
*/
- public void setOOBData(boolean expr_tree, boolean rewritten_expr_tree,
- boolean logical_plan, boolean optimized_logical_plan,
- boolean hyracks_job) {
+ public void setOOBData(boolean expr_tree, boolean rewritten_expr_tree, boolean logical_plan,
+ boolean optimized_logical_plan, boolean hyracks_job) {
this.set(OOB_EXPR_TREE, expr_tree);
this.set(OOB_REWRITTEN_EXPR_TREE, rewritten_expr_tree);
this.set(OOB_LOGICAL_PLAN, logical_plan);
@@ -192,8 +226,11 @@ public class SessionConfig {
/**
* Specify a flag.
- * @param flag One of the OOB_ or FORMAT_ constants from this class.
- * @param value Value for the flag (all flags default to "false").
+ *
+ * @param flag
+ * One of the OOB_ or FORMAT_ constants from this class.
+ * @param value
+ * Value for the flag (all flags default to "false").
*/
public void set(String flag, boolean value) {
flags.put(flag, Boolean.valueOf(value));
@@ -201,7 +238,9 @@ public class SessionConfig {
/**
* Retrieve the setting of a format-specific flag.
- * @param flag One of the FORMAT_ constants from this class.
+ *
+ * @param flag
+ * One of the FORMAT_ constants from this class.
* @returns true or false (all flags default to "false").
*/
public boolean is(String flag) {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fa8a7ade/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/JSONUtil.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/JSONUtil.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/JSONUtil.java
new file mode 100644
index 0000000..56f349c
--- /dev/null
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/JSONUtil.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.api.http.servlet;
+
+import java.util.Iterator;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+public class JSONUtil {
+
+ static final String INDENT = " ";
+
+ public static String indent(String str) {
+ try {
+ return append(new StringBuilder(), new JSONObject(str), 0).toString();
+ } catch (JSONException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ static StringBuilder append(StringBuilder sb, Object o, int indent) throws JSONException {
+ if (o instanceof JSONObject) {
+ return append(sb, (JSONObject) o, indent);
+ } else if (o instanceof JSONArray) {
+ return append(sb, (JSONArray) o, indent);
+ } else if (o instanceof String) {
+ return quote(sb, (String) o);
+ } else if (o instanceof Number || o instanceof Boolean) {
+ return sb.append(String.valueOf(o));
+ }
+ throw new UnsupportedOperationException(o.getClass().getSimpleName());
+ }
+
+ static StringBuilder append(StringBuilder sb, JSONObject jobj, int indent) throws JSONException {
+ sb = sb.append("{\n");
+ boolean first = true;
+ for (Iterator it = jobj.keys(); it.hasNext();) {
+ final String key = (String) it.next();
+ if (first) {
+ first = false;
+ } else {
+ sb = sb.append(",\n");
+ }
+ sb = indent(sb, indent + 1);
+ sb = quote(sb, key);
+ sb = sb.append(": ");
+ sb = append(sb, jobj.get(key), indent + 1);
+ }
+ sb = sb.append("\n");
+ return indent(sb, indent).append("}");
+ }
+
+ static StringBuilder append(StringBuilder sb, JSONArray jarr, int indent) throws JSONException {
+ sb = sb.append("[\n");
+ for (int i = 0; i < jarr.length(); ++i) {
+ if (i > 0) {
+ sb = sb.append(",\n");
+ }
+ sb = indent(sb, indent + 1);
+ sb = append(sb, jarr.get(i), indent + 1);
+ }
+ sb = sb.append("\n");
+ return indent(sb, indent).append("]");
+ }
+
+ static StringBuilder quote(StringBuilder sb, String str) {
+ return sb.append('"').append(str).append('"');
+ }
+
+ static StringBuilder indent(StringBuilder sb, int indent) {
+ while (indent > 0) {
+ sb.append(INDENT);
+ --indent;
+ }
+ return sb;
+ }
+
+ public static String escape(String str) {
+ StringBuilder result = new StringBuilder();
+ for (int i = 0; i < str.length(); ++i) {
+ appendEsc(result, str.charAt(i));
+ }
+ return result.toString();
+ }
+
+ public static StringBuilder appendEsc(StringBuilder sb, char c) {
+ switch (c) {
+ case '"':
+ return sb.append("\\\"");
+ case '\\':
+ return sb.append("\\\\");
+ case '/':
+ return sb.append("\\/");
+ case '\b':
+ return sb.append("\\b");
+ case '\n':
+ return sb.append("\\n");
+ case '\f':
+ return sb.append("\\f");
+ case '\r':
+ return sb.append("\\r");
+ case '\t':
+ return sb.append("\\t");
+ default:
+ return sb.append(c);
+ }
+ }
+
+ public static void main(String[] args) {
+ String json = args.length > 0 ? args[0] : "{\"a\":[\"b\",\"c\\\nd\"],\"e\":42}";
+ System.out.println(json);
+ System.out.println(indent(json));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fa8a7ade/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
index 3f7e9b5..d150e5d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
@@ -26,9 +26,6 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.json.JSONArray;
-import org.json.JSONObject;
-
import org.apache.asterix.api.common.SessionConfig;
import org.apache.asterix.result.ResultReader;
import org.apache.asterix.result.ResultUtils;
@@ -38,6 +35,8 @@ import org.apache.hyracks.api.dataset.IHyracksDataset;
import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.client.dataset.HyracksDataset;
+import org.json.JSONArray;
+import org.json.JSONObject;
public class QueryResultAPIServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
@@ -89,7 +88,7 @@ public class QueryResultAPIServlet extends HttpServlet {
// originally determined there. Need to save this value on
// some object that we can obtain here.
SessionConfig sessionConfig = RESTAPIServlet.initResponse(request, response);
- ResultUtils.displayResults(resultReader, sessionConfig);
+ ResultUtils.displayResults(resultReader, sessionConfig, new ResultUtils.Stats());
} catch (Exception e) {
out.println(e.getMessage());
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fa8a7ade/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
new file mode 100644
index 0000000..c4d270b
--- /dev/null
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
@@ -0,0 +1,342 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.api.http.servlet;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.UUID;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.asterix.api.common.SessionConfig;
+import org.apache.asterix.aql.translator.QueryTranslator;
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
+import org.apache.asterix.lang.aql.parser.TokenMgrError;
+import org.apache.asterix.lang.common.base.IParser;
+import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.result.ResultReader;
+import org.apache.asterix.result.ResultUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.hyracks.api.client.IHyracksClientConnection;
+import org.apache.hyracks.api.dataset.IHyracksDataset;
+import org.apache.hyracks.client.dataset.HyracksDataset;
+
+public class QueryServiceServlet extends HttpServlet {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOGGER = Logger.getLogger(QueryServiceServlet.class.getName());
+
+ public static final String HYRACKS_CONNECTION_ATTR = "org.apache.asterix.HYRACKS_CONNECTION";
+ public static final String HYRACKS_DATASET_ATTR = "org.apache.asterix.HYRACKS_DATASET";
+
+ public enum Parameter {
+ // Standard
+ statement,
+ format,
+ // Asterix
+ header
+ }
+
+ public enum Header {
+ Accept("Accept"),
+ ContentLength("Content-Length");
+
+ private final String str;
+
+ Header(String str) {
+ this.str = str;
+ }
+
+ public String str() {
+ return str;
+ }
+ }
+
+ public enum MediaType {
+ CSV("text/csv"),
+ JSON("application/json");
+
+ private final String str;
+
+ MediaType(String str) {
+ this.str = str;
+ }
+
+ public String str() {
+ return str;
+ }
+ }
+
+ public enum ResultFields {
+ requestID,
+ signature,
+ status,
+ results,
+ errors,
+ metrics
+ }
+
+ public enum ResultStatus {
+ success,
+ timeout,
+ errors,
+ fatal
+ }
+
+ public enum ErrorField {
+ code,
+ msg,
+ stack
+ }
+
+ public enum Metrics {
+ elapsedTime,
+ executionTime,
+ resultCount,
+ resultSize
+ }
+
+ private final ILangCompilationProvider compilationProvider = new SqlppCompilationProvider();
+
+ static SessionConfig.OutputFormat getFormat(HttpServletRequest request) {
+ // First check the "format" parameter.
+ String format = request.getParameter(Parameter.format.name());
+ if (format != null && format.equals("CSV")) {
+ return SessionConfig.OutputFormat.CSV;
+ }
+ // Second check the Accept: HTTP header.
+ String accept = request.getHeader(Header.Accept.str());
+ if (accept != null && accept.contains(MediaType.CSV.str())) {
+ return SessionConfig.OutputFormat.CSV;
+ }
+ return SessionConfig.OutputFormat.CLEAN_JSON;
+ }
+
+ /**
+ * Construct a SessionConfig with the appropriate output writer and
+ * output-format based on the Accept: header and other servlet parameters.
+ */
+ static SessionConfig createSessionConfig(HttpServletRequest request, PrintWriter resultWriter) {
+ SessionConfig.ResultDecorator resultPrefix = new SessionConfig.ResultDecorator() {
+ @Override
+ public PrintWriter print(PrintWriter pw) {
+ pw.print("\t\"");
+ pw.print(ResultFields.results.name());
+ pw.print("\": ");
+ return pw;
+ }
+ };
+
+ SessionConfig.ResultDecorator resultPostfix = new SessionConfig.ResultDecorator() {
+ @Override
+ public PrintWriter print(PrintWriter pw) {
+ pw.print(",\n");
+ return pw;
+ }
+ };
+
+ SessionConfig.OutputFormat format = getFormat(request);
+ SessionConfig sessionConfig = new SessionConfig(resultWriter, format, resultPrefix, resultPostfix);
+ sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, (format == SessionConfig.OutputFormat.CLEAN_JSON));
+
+ if (format == SessionConfig.OutputFormat.CSV && ("present".equals(request.getParameter(Parameter.header.name()))
+ || request.getHeader(Header.Accept.str()).contains("header=present"))) {
+ sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, true);
+ }
+ return sessionConfig;
+ }
+
+ /**
+ * Initialize the Content-Type of the response based on a SessionConfig.
+ */
+ static void initResponse(HttpServletResponse response, SessionConfig sessionConfig) throws IOException {
+ response.setCharacterEncoding("utf-8");
+ switch (sessionConfig.fmt()) {
+ case CLEAN_JSON:
+ response.setContentType(MediaType.JSON.str());
+ break;
+ case CSV:
+ String contentType = MediaType.CSV.str() + "; header="
+ + (sessionConfig.is(SessionConfig.FORMAT_CSV_HEADER) ? "present" : "absent");
+ response.setContentType(contentType);
+ break;
+ }
+ }
+
+ static void printField(PrintWriter pw, String name, String value) {
+ printField(pw, name, value, true);
+ }
+
+ static void printField(PrintWriter pw, String name, String value, boolean comma) {
+ pw.print("\t\"");
+ pw.print(name);
+ pw.print("\": \"");
+ pw.print(value);
+ pw.print('"');
+ if (comma) {
+ pw.print(',');
+ }
+ pw.print('\n');
+ }
+
+ static UUID printRequestId(PrintWriter pw) {
+ UUID requestId = UUID.randomUUID();
+ printField(pw, ResultFields.requestID.name(), requestId.toString());
+ return requestId;
+ }
+
+ static void printSignature(PrintWriter pw) {
+ printField(pw, ResultFields.signature.name(), "*");
+ }
+
+ static void printStatus(PrintWriter pw, ResultStatus rs) {
+ printField(pw, ResultFields.status.name(), rs.name());
+ }
+
+ static void printError(PrintWriter pw, Throwable e) {
+ final boolean addStack = false;
+ pw.print("\t\"");
+ pw.print(ResultFields.errors.name());
+ pw.print("\": [{ \n");
+ printField(pw, ErrorField.code.name(), "1");
+ printField(pw, ErrorField.msg.name(), JSONUtil.escape(e.getMessage()), addStack);
+ if (addStack) {
+ StringWriter sw = new StringWriter();
+ PrintWriter stackWriter = new PrintWriter(sw);
+ e.printStackTrace(stackWriter);
+ stackWriter.close();
+ printField(pw, ErrorField.stack.name(), JSONUtil.escape(sw.toString()), false);
+ }
+ pw.print("\t}],\n");
+ }
+
+ static void printMetrics(PrintWriter pw, long elapsedTime, long executionTime, long resultCount, long resultSize) {
+ pw.print("\t\"");
+ pw.print(ResultFields.metrics.name());
+ pw.print("\": {\n");
+ pw.print("\t");
+ printField(pw, Metrics.elapsedTime.name(), String.valueOf(elapsedTime));
+ pw.print("\t");
+ printField(pw, Metrics.executionTime.name(), String.valueOf(executionTime));
+ pw.print("\t");
+ printField(pw, Metrics.resultCount.name(), String.valueOf(resultCount));
+ pw.print("\t");
+ printField(pw, Metrics.resultSize.name(), String.valueOf(resultSize), false);
+ pw.print("\t}\n");
+ }
+
+ @Override
+ protected void doPost(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+ String query = request.getParameter(Parameter.statement.name());
+ if (query == null) {
+ StringWriter sw = new StringWriter();
+ IOUtils.copy(request.getInputStream(), sw, StandardCharsets.UTF_8.name());
+ query = sw.toString();
+ }
+ handleRequest(request, response, query);
+ }
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
+ String query = request.getParameter(Parameter.statement.name());
+ handleRequest(request, response, query);
+ }
+
+ public void handleRequest(HttpServletRequest request, HttpServletResponse response, String query)
+ throws IOException {
+ long elapsedStart = System.nanoTime();
+
+ query = query + ";";
+
+ final StringWriter stringWriter = new StringWriter();
+ final PrintWriter resultWriter = new PrintWriter(stringWriter);
+
+ SessionConfig sessionConfig = createSessionConfig(request, resultWriter);
+ initResponse(response, sessionConfig);
+
+ int respCode = HttpServletResponse.SC_OK;
+ ResultUtils.Stats stats = new ResultUtils.Stats();
+ long execStart = 0, execEnd = 0;
+
+ resultWriter.print("{\n");
+ UUID requestId = printRequestId(resultWriter);
+ printSignature(resultWriter);
+ try {
+ IHyracksClientConnection hcc;
+ IHyracksDataset hds;
+ ServletContext context = getServletContext();
+ synchronized (context) {
+ hcc = (IHyracksClientConnection) context.getAttribute(HYRACKS_CONNECTION_ATTR);
+ hds = (IHyracksDataset) context.getAttribute(HYRACKS_DATASET_ATTR);
+ if (hds == null) {
+ hds = new HyracksDataset(hcc, ResultReader.FRAME_SIZE, ResultReader.NUM_READERS);
+ context.setAttribute(HYRACKS_DATASET_ATTR, hds);
+ }
+ }
+ IParser parser = compilationProvider.getParserFactory().createParser(query);
+ List<Statement> aqlStatements = parser.parse();
+ MetadataManager.INSTANCE.init();
+ QueryTranslator translator = new QueryTranslator(aqlStatements, sessionConfig, compilationProvider);
+ execStart = System.nanoTime();
+ translator.compileAndExecute(hcc, hds, QueryTranslator.ResultDelivery.SYNC, stats);
+ execEnd = System.nanoTime();
+ printStatus(resultWriter, ResultStatus.success);
+ } catch (AsterixException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) {
+ GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, pe.getMessage(), pe);
+ printError(resultWriter, pe);
+ printStatus(resultWriter, ResultStatus.fatal);
+ respCode = HttpServletResponse.SC_BAD_REQUEST;
+ } catch (Exception e) {
+ GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e);
+ printError(resultWriter, e);
+ printStatus(resultWriter, ResultStatus.fatal);
+ respCode = HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+ }
+ printMetrics(resultWriter, (System.nanoTime() - elapsedStart) / 1000, (execEnd - execStart) / 1000, stats.count,
+ stats.size);
+ resultWriter.print("}\n");
+ resultWriter.flush();
+ String result = stringWriter.toString();
+
+ GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, result);
+ //result = JSONUtil.indent(result);
+
+ response.setIntHeader(Header.ContentLength.str(), result.length());
+ response.getWriter().print(result);
+ if (response.getWriter().checkError()) {
+ LOGGER.warning("Error flushing output writer");
+ }
+ response.setStatus(respCode);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fa8a7ade/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 4adadf9..46ea72b 100644
--- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -203,7 +203,7 @@ public class QueryTranslator extends AbstractLangTranslator {
ADDED_PENDINGOP_RECORD_TO_METADATA
}
- public static enum ResultDelivery {
+ public enum ResultDelivery {
SYNC,
ASYNC,
ASYNC_DEFERRED
@@ -238,6 +238,7 @@ public class QueryTranslator extends AbstractLangTranslator {
/**
* Compiles and submits for execution a list of AQL statements.
+ *
* @param hcc
* A Hyracks client connection that is used to submit a jobspec to Hyracks.
* @param hdc
@@ -249,6 +250,11 @@ public class QueryTranslator extends AbstractLangTranslator {
*/
public void compileAndExecute(IHyracksClientConnection hcc, IHyracksDataset hdc, ResultDelivery resultDelivery)
throws Exception {
+ compileAndExecute(hcc, hdc, resultDelivery, new ResultUtils.Stats());
+ }
+
+ public void compileAndExecute(IHyracksClientConnection hcc, IHyracksDataset hdc, ResultDelivery resultDelivery,
+ ResultUtils.Stats stats) throws Exception {
int resultSetIdCounter = 0;
FileSplit outputFile = null;
IAWriterFactory writerFactory = PrinterBasedWriterFactory.INSTANCE;
@@ -381,7 +387,7 @@ public class QueryTranslator extends AbstractLangTranslator {
metadataProvider.setResultSetId(new ResultSetId(resultSetIdCounter++));
metadataProvider.setResultAsyncMode(
resultDelivery == ResultDelivery.ASYNC || resultDelivery == ResultDelivery.ASYNC_DEFERRED);
- handleQuery(metadataProvider, (Query) stmt, hcc, hdc, resultDelivery);
+ handleQuery(metadataProvider, (Query) stmt, hcc, hdc, resultDelivery, stats);
break;
}
@@ -2212,6 +2218,7 @@ public class QueryTranslator extends AbstractLangTranslator {
/**
* Generates a subscription request corresponding to a connect feed request. In addition, provides a boolean
* flag indicating if feed intake job needs to be started (source primary feed not found to be active).
+ *
* @param dataverse
* @param feed
* @param dataset
@@ -2483,7 +2490,7 @@ public class QueryTranslator extends AbstractLangTranslator {
}
private void handleQuery(AqlMetadataProvider metadataProvider, Query query, IHyracksClientConnection hcc,
- IHyracksDataset hdc, ResultDelivery resultDelivery) throws Exception {
+ IHyracksDataset hdc, ResultDelivery resultDelivery, ResultUtils.Stats stats) throws Exception {
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
boolean bActiveTxn = true;
@@ -2523,7 +2530,7 @@ public class QueryTranslator extends AbstractLangTranslator {
&& sessionConfig.is(SessionConfig.FORMAT_CSV_HEADER)) {
ResultUtils.displayCSVHeader(metadataProvider.findOutputRecordType(), sessionConfig);
}
- ResultUtils.displayResults(resultReader, sessionConfig);
+ ResultUtils.displayResults(resultReader, sessionConfig, stats);
break;
case ASYNC_DEFERRED:
handle = new JSONArray();
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fa8a7ade/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
index f2fc9bf..bee284d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
@@ -28,6 +28,7 @@ import org.apache.asterix.api.http.servlet.DDLAPIServlet;
import org.apache.asterix.api.http.servlet.FeedServlet;
import org.apache.asterix.api.http.servlet.QueryAPIServlet;
import org.apache.asterix.api.http.servlet.QueryResultAPIServlet;
+import org.apache.asterix.api.http.servlet.QueryServiceServlet;
import org.apache.asterix.api.http.servlet.QueryStatusAPIServlet;
import org.apache.asterix.api.http.servlet.ShutdownAPIServlet;
import org.apache.asterix.api.http.servlet.UpdateAPIServlet;
@@ -194,6 +195,8 @@ public class CCApplicationEntryPoint implements ICCApplicationEntryPoint {
context.addServlet(new ServletHolder(new ConnectorAPIServlet()), "/connector");
context.addServlet(new ServletHolder(new ShutdownAPIServlet()), "/admin/shutdown");
context.addServlet(new ServletHolder(new VersionAPIServlet()), "/admin/version");
+
+ context.addServlet(new ServletHolder(new QueryServiceServlet()), "/query/service");
}
private void setupFeedServer(AsterixExternalProperties externalProperties) throws Exception {
@@ -220,4 +223,4 @@ public class CCApplicationEntryPoint implements ICCApplicationEntryPoint {
ClusterState.ACTIVE);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fa8a7ade/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java b/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java
index 2f82f80..f1d20d0 100644
--- a/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java
+++ b/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java
@@ -58,6 +58,11 @@ public class ResultUtils {
HTML_ENTITIES.put('>', ">");
}
+ public static class Stats {
+ public long count;
+ public long size;
+ }
+
public static String escapeHTML(String s) {
for (Character c : HTML_ENTITIES.keySet()) {
if (s.indexOf(c) >= 0) {
@@ -91,7 +96,8 @@ public class ResultUtils {
public static FrameManager resultDisplayFrameMgr = new FrameManager(ResultReader.FRAME_SIZE);
- public static void displayResults(ResultReader resultReader, SessionConfig conf) throws HyracksDataException {
+ public static void displayResults(ResultReader resultReader, SessionConfig conf, Stats stats)
+ throws HyracksDataException {
IFrameTupleAccessor fta = resultReader.getFrameTupleAccessor();
IFrame frame = new VSizeFrame(resultDisplayFrameMgr);
@@ -111,6 +117,8 @@ public class ResultUtils {
conf.out().println("<pre>");
}
+ conf.resultPrefix(conf.out());
+
switch (conf.fmt()) {
case LOSSLESS_JSON:
case CLEAN_JSON:
@@ -153,6 +161,9 @@ public class ResultUtils {
if (conf.fmt() == OutputFormat.CSV) {
conf.out().print("\r\n");
}
+ ++stats.count;
+ // TODO(tillw) fix this approximation
+ stats.size += result.length();
}
frame.getBuffer().clear();
} finally {
@@ -171,6 +182,8 @@ public class ResultUtils {
conf.out().println(" ]");
}
+ conf.resultPostfix(conf.out());
+
if (conf.is(SessionConfig.FORMAT_HTML)) {
conf.out().println("</pre>");
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fa8a7ade/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 823e861..3e709bf 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -505,4 +505,4 @@ public class MetadataBootstrap {
MetadataManager.INSTANCE.releaseWriteLatch();
}
}
-}
\ No newline at end of file
+}