You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ti...@apache.org on 2016/09/24 00:50:26 UTC
asterixdb git commit: add support for JSON encoded requests
Repository: asterixdb
Updated Branches:
refs/heads/master 8bef7a3a8 -> 6762f03ec
add support for JSON encoded requests
- run SQL++ execution test *queries* using JSON encoded requests
- fix metadata cleanup at the end of the tests
Change-Id: I1cc934d5dd984b476d4adb1755572d2e2f451985
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1201
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <bu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/6762f03e
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/6762f03e
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/6762f03e
Branch: refs/heads/master
Commit: 6762f03ec4c35d951c0af7c69363ca9898afba6e
Parents: 8bef7a3
Author: Till Westmann <ti...@apache.org>
Authored: Fri Sep 23 14:41:57 2016 -0700
Committer: Till Westmann <ti...@apache.org>
Committed: Fri Sep 23 17:49:57 2016 -0700
----------------------------------------------------------------------
.../api/http/servlet/QueryServiceServlet.java | 90 ++++++++++++++------
.../apache/asterix/test/aql/TestExecutor.java | 79 ++++++++++-------
2 files changed, 111 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6762f03e/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
index 132737b..856aa40 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
@@ -56,6 +56,8 @@ import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendab
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.dataset.IHyracksDataset;
import org.apache.hyracks.client.dataset.HyracksDataset;
+import org.json.JSONException;
+import org.json.JSONObject;
public class QueryServiceServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
@@ -71,10 +73,9 @@ public class QueryServiceServlet extends HttpServlet {
}
public enum Parameter {
- // Standard
STATEMENT("statement"),
FORMAT("format"),
- // Asterix
+ CLIENT_ID("client_context_id"),
PRETTY("pretty");
private final String str;
@@ -121,6 +122,7 @@ public class QueryServiceServlet extends HttpServlet {
public enum ResultFields {
REQUEST_ID("requestID"),
+ CLIENT_ID("clientContextID"),
SIGNATURE("signature"),
TYPE("type"),
STATUS("status"),
@@ -217,6 +219,13 @@ public class QueryServiceServlet extends HttpServlet {
}
}
+ static class RequestParameters {
+ String statement;
+ String format;
+ boolean pretty;
+ String clientContextID;
+ }
+
private static String getParameterValue(String content, String attribute) {
if (content == null || attribute == null) {
return null;
@@ -255,11 +264,7 @@ public class QueryServiceServlet extends HttpServlet {
return SessionConfig.OutputFormat.CLEAN_JSON;
}
- /**
- * Construct a SessionConfig with the appropriate output writer and
- * output-format based on the Accept: header and other servlet parameters.
- */
- private static SessionConfig createSessionConfig(HttpServletRequest request, PrintWriter resultWriter) {
+ private static SessionConfig createSessionConfig(RequestParameters param, PrintWriter resultWriter) {
SessionConfig.ResultDecorator resultPrefix = (AlgebricksAppendable app) -> {
app.append("\t\"");
app.append(ResultFields.RESULTS.str());
@@ -272,16 +277,14 @@ public class QueryServiceServlet extends HttpServlet {
return app;
};
- final String formatstr = toLower(request.getParameter(Parameter.FORMAT.str()));
- SessionConfig.OutputFormat format = getFormat(formatstr);
+ SessionConfig.OutputFormat format = getFormat(param.format);
SessionConfig sessionConfig = new SessionConfig(resultWriter, format, resultPrefix, resultPostfix);
sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, true);
- boolean indentJson = Boolean.parseBoolean(request.getParameter(Parameter.PRETTY.str()));
- sessionConfig.set(SessionConfig.FORMAT_INDENT_JSON, indentJson);
+ sessionConfig.set(SessionConfig.FORMAT_INDENT_JSON, param.pretty);
sessionConfig.set(SessionConfig.FORMAT_QUOTE_RECORD,
format != SessionConfig.OutputFormat.CLEAN_JSON && format != SessionConfig.OutputFormat.LOSSLESS_JSON);
- sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER,
- format == SessionConfig.OutputFormat.CSV && "present".equals(getParameterValue(formatstr, "header")));
+ sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, format == SessionConfig.OutputFormat.CSV
+ && "present".equals(getParameterValue(param.format, "header")));
return sessionConfig;
}
@@ -307,6 +310,12 @@ public class QueryServiceServlet extends HttpServlet {
return requestId;
}
+ private static void printClientContextID(PrintWriter pw, RequestParameters params) {
+ if (params.clientContextID != null && !params.clientContextID.isEmpty()) {
+ printField(pw, ResultFields.CLIENT_ID.str(), params.clientContextID);
+ }
+ }
+
private static void printSignature(PrintWriter pw) {
printField(pw, ResultFields.SIGNATURE.str(), "*");
}
@@ -370,16 +379,9 @@ public class QueryServiceServlet extends HttpServlet {
}
@Override
- protected void doPost(HttpServletRequest request, HttpServletResponse response)
- throws ServletException, IOException {
- String query = request.getParameter(Parameter.STATEMENT.str());
+ protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException {
try {
- if (query == null) {
- StringWriter sw = new StringWriter();
- IOUtils.copy(request.getInputStream(), sw, StandardCharsets.UTF_8.name());
- query = sw.toString();
- }
- handleRequest(request, response, query);
+ handleRequest(getRequestParameters(request), response);
} catch (IOException e) {
// Servlet methods should not throw exceptions
// http://cwe.mitre.org/data/definitions/600.html
@@ -387,13 +389,46 @@ public class QueryServiceServlet extends HttpServlet {
}
}
- private void handleRequest(HttpServletRequest request, HttpServletResponse response, String query)
- throws IOException {
+ private RequestParameters getRequestParameters(HttpServletRequest request) throws IOException {
+ final String contentTypeParam = request.getContentType();
+ int sep = contentTypeParam.indexOf(';');
+ final String contentType = sep < 0 ? contentTypeParam.trim() : contentTypeParam.substring(0, sep).trim();
+ RequestParameters param = new RequestParameters();
+ if (MediaType.JSON.str().equals(contentType)) {
+ try {
+ JSONObject jsonRequest = new JSONObject(getRequestBody(request));
+ param.statement = jsonRequest.getString(Parameter.STATEMENT.str());
+ param.format = toLower(jsonRequest.optString(Parameter.FORMAT.str()));
+ param.pretty = jsonRequest.optBoolean(Parameter.PRETTY.str());
+ param.clientContextID = jsonRequest.optString(Parameter.CLIENT_ID.str());
+ } catch (JSONException e) {
+ // if the JSON parsing fails, the statement is empty and we get an empty statement error
+ GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e);
+ }
+ } else {
+ param.statement = request.getParameter(Parameter.STATEMENT.str());
+ if (param.statement == null) {
+ param.statement = getRequestBody(request);
+ }
+ param.format = toLower(request.getParameter(Parameter.FORMAT.str()));
+ param.pretty = Boolean.parseBoolean(request.getParameter(Parameter.PRETTY.str()));
+ param.clientContextID = request.getParameter(Parameter.CLIENT_ID.str());
+ }
+ return param;
+ }
+
+ private static String getRequestBody(HttpServletRequest request) throws IOException {
+ StringWriter sw = new StringWriter();
+ IOUtils.copy(request.getInputStream(), sw, StandardCharsets.UTF_8.name());
+ return sw.toString();
+ }
+
+ private void handleRequest(RequestParameters param, HttpServletResponse response) throws IOException {
long elapsedStart = System.nanoTime();
final StringWriter stringWriter = new StringWriter();
final PrintWriter resultWriter = new PrintWriter(stringWriter);
- SessionConfig sessionConfig = createSessionConfig(request, resultWriter);
+ SessionConfig sessionConfig = createSessionConfig(param, resultWriter);
response.setCharacterEncoding("utf-8");
response.setContentType(MediaType.JSON.str());
@@ -404,10 +439,11 @@ public class QueryServiceServlet extends HttpServlet {
resultWriter.print("{\n");
printRequestId(resultWriter);
+ printClientContextID(resultWriter, param);
printSignature(resultWriter);
printType(resultWriter, sessionConfig);
try {
- if (query == null || query.isEmpty()) {
+ if (param.statement == null || param.statement.isEmpty()) {
throw new AsterixException("Empty request, no statement provided");
}
IHyracksClientConnection hcc;
@@ -421,7 +457,7 @@ public class QueryServiceServlet extends HttpServlet {
context.setAttribute(HYRACKS_DATASET_ATTR, hds);
}
}
- IParser parser = compilationProvider.getParserFactory().createParser(query);
+ IParser parser = compilationProvider.getParserFactory().createParser(param.statement);
List<Statement> aqlStatements = parser.parse();
MetadataManager.INSTANCE.init();
IStatementExecutor translator = statementExecutorFactory.create(aqlStatements, sessionConfig,
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6762f03e/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
index 1c48dbb..08a0342 100644
--- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
@@ -60,10 +60,13 @@ import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
+import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
import org.apache.http.util.EntityUtils;
+import org.json.JSONArray;
+import org.json.JSONException;
import org.json.JSONObject;
public class TestExecutor {
@@ -347,9 +350,7 @@ public class TestExecutor {
}
protected HttpResponse executeHttpRequest(HttpUriRequest method) throws Exception {
- HttpClient client = HttpClients.custom()
- .setRetryHandler(StandardHttpRequestRetryHandler.INSTANCE)
- .build();
+ HttpClient client = HttpClients.custom().setRetryHandler(StandardHttpRequestRetryHandler.INSTANCE).build();
try {
return client.execute(method);
} catch (Exception e) {
@@ -395,13 +396,14 @@ public class TestExecutor {
}
public InputStream executeQueryService(String str, String url) throws Exception {
- return executeQueryService(str, OutputFormat.CLEAN_JSON, url, new ArrayList<>());
+ return executeQueryService(str, OutputFormat.CLEAN_JSON, url, new ArrayList<>(), false);
}
public InputStream executeQueryService(String str, OutputFormat fmt, String url,
- List<CompilationUnit.Parameter> params) throws Exception {
+ List<CompilationUnit.Parameter> params, boolean jsonEncoded) throws Exception {
setFormatParam(params, fmt);
- HttpUriRequest method = constructPostMethod(str, url, "statement", true, params);
+ HttpUriRequest method = jsonEncoded ? constructPostMethodJson(str, url, "statement", params)
+ : constructPostMethodUrl(str, url, "statement", params);
// Set accepted output response type
method.setHeader("Accept", OutputFormat.CLEAN_JSON.mimeType());
HttpResponse response = executeHttpRequest(method);
@@ -409,19 +411,16 @@ public class TestExecutor {
}
protected void setFormatParam(List<CompilationUnit.Parameter> params, OutputFormat fmt) {
- boolean formatSet = false;
for (CompilationUnit.Parameter param : params) {
if ("format".equals(param.getName())) {
param.setValue(fmt.mimeType());
- formatSet = true;
+ return;
}
}
- if (!formatSet) {
- CompilationUnit.Parameter formatParam = new CompilationUnit.Parameter();
- formatParam.setName("format");
- formatParam.setValue(fmt.mimeType());
- params.add(formatParam);
- }
+ CompilationUnit.Parameter formatParam = new CompilationUnit.Parameter();
+ formatParam.setName("format");
+ formatParam.setValue(fmt.mimeType());
+ params.add(formatParam);
}
private HttpUriRequest constructHttpMethod(String statement, String endpoint, String stmtParam,
@@ -431,7 +430,8 @@ public class TestExecutor {
return constructGetMethod(statement, endpoint, stmtParam, otherParams);
} else {
// Use POST for bigger ones to avoid 413 FULL_HEAD
- return constructPostMethod(statement, endpoint, stmtParam, postStmtAsParam, otherParams);
+ String stmtParamName = (postStmtAsParam ? stmtParam : null);
+ return constructPostMethodUrl(statement, endpoint, stmtParamName, otherParams);
}
}
@@ -445,10 +445,10 @@ public class TestExecutor {
return builder.build();
}
- protected HttpUriRequest constructPostMethod(String statement, String endpoint, String stmtParam,
- boolean postStmtAsParam, List<CompilationUnit.Parameter> otherParams) {
+ protected HttpUriRequest constructPostMethodUrl(String statement, String endpoint, String stmtParam,
+ List<CompilationUnit.Parameter> otherParams) {
RequestBuilder builder = RequestBuilder.post(endpoint);
- if (postStmtAsParam) {
+ if (stmtParam != null) {
for (CompilationUnit.Parameter param : otherParams) {
builder.addParameter(param.getName(), param.getValue());
}
@@ -461,6 +461,26 @@ public class TestExecutor {
return builder.build();
}
+ protected HttpUriRequest constructPostMethodJson(String statement, String endpoint, String stmtParam,
+ List<CompilationUnit.Parameter> otherParams) {
+ if (stmtParam == null) {
+ throw new NullPointerException("Statement parameter required.");
+ }
+ RequestBuilder builder = RequestBuilder.post(endpoint);
+ JSONObject content = new JSONObject();
+ try {
+ content.put(stmtParam, statement);
+ for (CompilationUnit.Parameter param : otherParams) {
+ content.put(param.getName(), param.getValue());
+ }
+ } catch (JSONException e) {
+ throw new IllegalArgumentException("Request object construction failed.", e);
+ }
+ builder.setEntity(new StringEntity(content.toString(), ContentType.APPLICATION_JSON));
+ builder.setCharset(StandardCharsets.UTF_8);
+ return builder.build();
+ }
+
public InputStream executeClusterStateQuery(OutputFormat fmt, String url) throws Exception {
HttpUriRequest request = RequestBuilder.get(url).setHeader("Accept", fmt.mimeType()).build();
@@ -485,9 +505,7 @@ public class TestExecutor {
// Create a method instance.
HttpUriRequest request = RequestBuilder.post(url)
.addParameter("mode", defer ? "asynchronous-deferred" : "asynchronous")
- .setEntity(new StringEntity(str, StandardCharsets.UTF_8))
- .setHeader("Accept", fmt.mimeType())
- .build();
+ .setEntity(new StringEntity(str, StandardCharsets.UTF_8)).setHeader("Accept", fmt.mimeType()).build();
HttpResponse response = executeAndCheckHttpRequest(request);
InputStream resultStream = response.getEntity().getContent();
@@ -664,7 +682,7 @@ public class TestExecutor {
} else {
if (ctx.getType().equalsIgnoreCase("query")) {
resultStream = executeQueryService(statement, fmt, getEndpoint(Servlets.QUERY_SERVICE),
- cUnit.getParameter());
+ cUnit.getParameter(), true);
resultStream = ResultExtractor.extract(resultStream);
} else if (ctx.getType().equalsIgnoreCase("async")) {
resultStream = executeAnyAQLAsync(statement, false, fmt, getEndpoint(Servlets.SQLPP));
@@ -968,20 +986,18 @@ public class TestExecutor {
public void cleanup(String testCase, List<String> badtestcases) throws Exception {
try {
ArrayList<String> toBeDropped = new ArrayList<>();
- InputStream resultStream = null;
- OutputFormat fmt = OutputFormat.ADM;
- resultStream = executeQueryService("select dv.DataverseName from Metadata.`Dataverse` as dv;", fmt,
- getEndpoint(Servlets.QUERY_SERVICE), new ArrayList<>());
+ InputStream resultStream = executeQueryService("select dv.DataverseName from Metadata.`Dataverse` as dv;",
+ getEndpoint(Servlets.QUERY_SERVICE));
resultStream = ResultExtractor.extract(resultStream);
- BufferedReader reader = new BufferedReader(new InputStreamReader(resultStream));
- String dataverse = reader.readLine();
- while (dataverse != null) {
- JSONObject json = new JSONObject(dataverse);
+ StringWriter sw = new StringWriter();
+ IOUtils.copy(resultStream, sw, StandardCharsets.UTF_8.name());
+ JSONArray result = new JSONArray(sw.toString());
+ for (int i = 0; i < result.length(); ++i) {
+ JSONObject json = result.getJSONObject(i);
String dvName = json.getString("DataverseName");
if (!dvName.equals("Metadata") && !dvName.equals("Default")) {
toBeDropped.add(dvName);
}
- dataverse = reader.readLine();
}
if (!toBeDropped.isEmpty()) {
badtestcases.add(testCase);
@@ -998,6 +1014,7 @@ public class TestExecutor {
}
} catch (Throwable th) {
th.printStackTrace();
+ throw th;
}
}
}