You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by "Till Westmann (Code Review)" <do...@asterixdb.incubator.apache.org> on 2017/03/05 02:57:35 UTC

Change in asterixdb[master]: WIP - More consistency between the HTTP APIs

Till Westmann has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/1550

Change subject: WIP - More consistency between the HTTP APIs
......................................................................

WIP - More consistency between the HTTP APIs

Change-Id: Ie0e58cac20c1976610f38796d06f0518a3174c50
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java
M asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-failed/async-failed.2.json
M asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.2.json
D asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.3.json
A asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.3.regex
D asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async/async.2.json
A asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async/async.2.regex
D asterixdb/asterix-app/src/test/resources/runtimets/results/flwor/at00/at00.5.json
A asterixdb/asterix-app/src/test/resources/runtimets/results/flwor/at00/at00.5.regex
M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/DatasetJobRecord.java
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/web/util/JSONOutputRequestHandler.java
M hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobStatusAPIIntegrationTest.java
14 files changed, 148 insertions(+), 104 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/50/1550/1

diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
index 8d934ee..79748f4 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
@@ -22,6 +22,8 @@
 import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
 
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.UUID;
 import java.util.concurrent.ConcurrentMap;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -32,6 +34,7 @@
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.dataset.IHyracksDataset;
 import org.apache.hyracks.client.dataset.HyracksDataset;
+import org.apache.hyracks.http.api.IServletRequest;
 import org.apache.hyracks.http.server.AbstractServlet;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -39,6 +42,46 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 class AbstractQueryApiServlet extends AbstractServlet {
+
+    public enum ResultFields {
+        REQUEST_ID("requestID"),
+        CLIENT_ID("clientContextID"),
+        SIGNATURE("signature"),
+        TYPE("type"),
+        STATUS("status"),
+        RESULTS("results"),
+        HANDLE("handle"),
+        ERRORS("errors"),
+        METRICS("metrics");
+
+        private final String str;
+
+        ResultFields(String str) {
+            this.str = str;
+        }
+
+        public String str() {
+            return str;
+        }
+    }
+
+    public enum ResultStatus {
+        STARTED("started"),
+        SUCCESS("success"),
+        TIMEOUT("timeout"),
+        FAILED("failed"),
+        FATAL("fatal");
+
+        private final String str;
+
+        ResultStatus(String str) {
+            this.str = str;
+        }
+
+        public String str() {
+            return str;
+        }
+    }
 
     AbstractQueryApiServlet(ConcurrentMap<String, Object> ctx, String[] paths) {
         super(ctx, paths);
@@ -82,4 +125,42 @@
         }
         return null;
     }
+
+    protected static UUID printRequestId(PrintWriter pw) {
+        UUID requestId = UUID.randomUUID();
+        printField(pw, ResultFields.REQUEST_ID.str(), requestId.toString());
+        return requestId;
+    }
+
+    protected static void printStatus(PrintWriter pw, ResultStatus rs) {
+        printField(pw, ResultFields.STATUS.str(), rs.str());
+    }
+
+    protected static void printHandle(PrintWriter pw, String handle) {
+        printField(pw, ResultFields.HANDLE.str(), handle);
+    }
+
+    protected static void printField(PrintWriter pw, String name, String value) {
+        printField(pw, name, value, true);
+    }
+
+    protected static void printField(PrintWriter pw, String name, String value, boolean comma) {
+        printFieldInternal(pw, name, "\"" + value + "\"", comma);
+    }
+
+    protected static void printField(PrintWriter pw, String name, long value, boolean comma) {
+        printFieldInternal(pw, name, String.valueOf(value), comma);
+    }
+
+    protected static void printFieldInternal(PrintWriter pw, String name, String value, boolean comma) {
+        pw.print("\t\"");
+        pw.print(name);
+        pw.print("\": ");
+        pw.print(value);
+        if (comma) {
+            pw.print(',');
+        }
+        pw.print('\n');
+    }
+
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index c0a38e8..f0772b4 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -69,7 +69,7 @@
 import io.netty.handler.codec.http.HttpMethod;
 import io.netty.handler.codec.http.HttpResponseStatus;
 
-public class QueryServiceServlet extends AbstractServlet {
+public class QueryServiceServlet extends AbstractQueryApiServlet {
     private static final Logger LOGGER = Logger.getLogger(QueryServiceServlet.class.getName());
     private final ILangCompilationProvider compilationProvider;
     private final IStatementExecutorFactory statementExecutorFactory;
@@ -120,46 +120,6 @@
         private final String str;
 
         Attribute(String str) {
-            this.str = str;
-        }
-
-        public String str() {
-            return str;
-        }
-    }
-
-    public enum ResultFields {
-        REQUEST_ID("requestID"),
-        CLIENT_ID("clientContextID"),
-        SIGNATURE("signature"),
-        TYPE("type"),
-        STATUS("status"),
-        RESULTS("results"),
-        HANDLE("handle"),
-        ERRORS("errors"),
-        METRICS("metrics");
-
-        private final String str;
-
-        ResultFields(String str) {
-            this.str = str;
-        }
-
-        public String str() {
-            return str;
-        }
-    }
-
-    public enum ResultStatus {
-        STARTED("started"),
-        SUCCESS("success"),
-        TIMEOUT("timeout"),
-        ERRORS("errors"),
-        FATAL("fatal");
-
-        private final String str;
-
-        ResultStatus(String str) {
             this.str = str;
         }
 
@@ -327,35 +287,6 @@
         return sessionConfig;
     }
 
-    private static void printField(PrintWriter pw, String name, String value) {
-        printField(pw, name, value, true);
-    }
-
-    private static void printField(PrintWriter pw, String name, String value, boolean comma) {
-        printFieldInternal(pw, name, "\"" + value + "\"", comma);
-    }
-
-    private static void printField(PrintWriter pw, String name, long value, boolean comma) {
-        printFieldInternal(pw, name, String.valueOf(value), comma);
-    }
-
-    private static void printFieldInternal(PrintWriter pw, String name, String value, boolean comma) {
-        pw.print("\t\"");
-        pw.print(name);
-        pw.print("\": ");
-        pw.print(value);
-        if (comma) {
-            pw.print(',');
-        }
-        pw.print('\n');
-    }
-
-    private static UUID printRequestId(PrintWriter pw) {
-        UUID requestId = UUID.randomUUID();
-        printField(pw, ResultFields.REQUEST_ID.str(), requestId.toString());
-        return requestId;
-    }
-
     private static void printClientContextID(PrintWriter pw, RequestParameters params) {
         if (params.clientContextID != null && !params.clientContextID.isEmpty()) {
             printField(pw, ResultFields.CLIENT_ID.str(), params.clientContextID);
@@ -379,10 +310,6 @@
             default:
                 break;
         }
-    }
-
-    private static void printStatus(PrintWriter pw, ResultStatus rs) {
-        printField(pw, ResultFields.STATUS.str(), rs.str());
     }
 
     private static void printError(PrintWriter pw, Throwable e) throws JsonProcessingException {
@@ -501,24 +428,13 @@
             if (param.statement == null || param.statement.isEmpty()) {
                 throw new AsterixException("Empty request, no statement provided");
             }
-            IHyracksClientConnection hcc = (IHyracksClientConnection) ctx.get(HYRACKS_CONNECTION_ATTR);
-            IHyracksDataset hds = (IHyracksDataset) ctx.get(HYRACKS_DATASET_ATTR);
-            if (hds == null) {
-                synchronized (ctx) {
-                    hds = (IHyracksDataset) ctx.get(HYRACKS_DATASET_ATTR);
-                    if (hds == null) {
-                        hds = new HyracksDataset(hcc, ResultReader.FRAME_SIZE, ResultReader.NUM_READERS);
-                        ctx.put(HYRACKS_DATASET_ATTR, hds);
-                    }
-                }
-            }
             IParser parser = compilationProvider.getParserFactory().createParser(param.statement);
             List<Statement> statements = parser.parse();
             MetadataManager.INSTANCE.init();
             IStatementExecutor translator =
                     statementExecutorFactory.create(statements, sessionConfig, compilationProvider, componentProvider);
             execStart = System.nanoTime();
-            translator.compileAndExecute(hcc, hds, delivery, stats);
+            translator.compileAndExecute(getHyracksClientConnection(), getHyracksDataset(), delivery, stats);
             execEnd = System.nanoTime();
             printStatus(resultWriter, ResultDelivery.ASYNC == delivery ? ResultStatus.STARTED : ResultStatus.SUCCESS);
         } catch (AsterixException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java
index 33c5c8f..56d985f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java
@@ -18,6 +18,8 @@
  */
 package org.apache.asterix.api.http.server;
 
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.util.concurrent.ConcurrentMap;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -33,7 +35,6 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
 
 import io.netty.handler.codec.http.HttpResponseStatus;
 
@@ -59,16 +60,55 @@
         IHyracksDataset hds = getHyracksDataset();
         ResultReader resultReader = new ResultReader(hds, jobId, rsId);
 
-        ObjectNode jsonResponse = om.createObjectNode();
-        final DatasetJobRecord.Status status = resultReader.getStatus();
-        if (status == null) {
+        ResultStatus resultStatus = resultStatus(resultReader.getStatus());
+
+        if (resultStatus == null) {
             LOGGER.log(Level.INFO, "No results for: \"" + strHandle + "\"");
             response.setStatus(HttpResponseStatus.NOT_FOUND);
             return;
         }
-        jsonResponse.put("status", status.name());
-        HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_PLAIN, HttpUtil.Encoding.UTF8);
-        response.setStatus(HttpResponseStatus.OK);
-        response.writer().write(jsonResponse.toString());
+
+        final StringWriter stringWriter = new StringWriter();
+        final PrintWriter resultWriter = new PrintWriter(stringWriter);
+
+        HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
+        HttpResponseStatus httpStatus = HttpResponseStatus.OK;
+
+        resultWriter.print("{\n");
+        //printRequestId(resultWriter);
+        printStatus(resultWriter, resultStatus);
+
+        if (ResultStatus.SUCCESS == resultStatus) {
+            String servletPath = servletPath(request).replace("status", "result");
+            String resHandle = "http://" + host(request) + servletPath + localPath(request);
+            printHandle(resultWriter, resHandle);
+        }
+
+        resultWriter.print("}\n");
+        resultWriter.flush();
+        String result = stringWriter.toString();
+
+        response.setStatus(httpStatus);
+        response.writer().print(result);
+        if (response.writer().checkError()) {
+            LOGGER.warning("Error flushing output writer");
+        }
+    }
+
+    ResultStatus resultStatus(DatasetJobRecord.Status status) {
+        if (status == null) {
+            return null;
+        }
+        switch (status) {
+            case IDLE:
+            case STARTED:
+                return ResultStatus.STARTED;
+            case SUCCESS:
+                return ResultStatus.SUCCESS;
+            case FAILED:
+                return ResultStatus.FAILED;
+            default:
+                return ResultStatus.FATAL;
+        }
     }
 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-failed/async-failed.2.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-failed/async-failed.2.json
index dd665eb..246785b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-failed/async-failed.2.json
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-failed/async-failed.2.json
@@ -1 +1,3 @@
-{"status":"FAILED"}
+{
+	"status": "failed",
+}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.2.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.2.json
index 6cffe65..9dbe8ce 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.2.json
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.2.json
@@ -1 +1,3 @@
-{"status":"RUNNING"}
+{
+	"status": "started",
+}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.3.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.3.json
deleted file mode 100644
index 6213a6b..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.3.json
+++ /dev/null
@@ -1 +0,0 @@
-{"status":"SUCCESS"}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.3.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.3.regex
new file mode 100644
index 0000000..4308ba2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async-running/async-running.3.regex
@@ -0,0 +1,2 @@
+/"status": "success"/
+/"handle": ".*"/
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async/async.2.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async/async.2.json
deleted file mode 100644
index 6213a6b..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async/async.2.json
+++ /dev/null
@@ -1 +0,0 @@
-{"status":"SUCCESS"}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async/async.2.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async/async.2.regex
new file mode 100644
index 0000000..4308ba2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred/async/async.2.regex
@@ -0,0 +1,2 @@
+/"status": "success"/
+/"handle": ".*"/
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/flwor/at00/at00.5.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/flwor/at00/at00.5.json
deleted file mode 100644
index 6213a6b..0000000
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/flwor/at00/at00.5.json
+++ /dev/null
@@ -1 +0,0 @@
-{"status":"SUCCESS"}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/flwor/at00/at00.5.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/flwor/at00/at00.5.regex
new file mode 100644
index 0000000..4308ba2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/flwor/at00/at00.5.regex
@@ -0,0 +1,2 @@
+/"status": "success"/
+/"handle": ".*"/
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/DatasetJobRecord.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/DatasetJobRecord.java
index f29ff4a..603351d 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/DatasetJobRecord.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/DatasetJobRecord.java
@@ -28,7 +28,7 @@
 public class DatasetJobRecord implements IDatasetStateRecord {
     public enum Status {
         IDLE,
-        RUNNING,
+        STARTED,
         SUCCESS,
         FAILED
     }
@@ -56,7 +56,7 @@
     }
 
     public void start() {
-        updateStatus(Status.RUNNING);
+        updateStatus(Status.STARTED);
     }
 
     public void success() {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/web/util/JSONOutputRequestHandler.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/web/util/JSONOutputRequestHandler.java
index b39915f..cadb219 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/web/util/JSONOutputRequestHandler.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/web/util/JSONOutputRequestHandler.java
@@ -48,11 +48,11 @@
         String localPath = localPath(request);
         String servletPath = servletPath(request);
         String host = host(request);
+
         while (localPath.startsWith("/")) {
             localPath = localPath.substring(1);
         }
         String[] parts = localPath.split("/");
-
         ObjectNode result = invoke(response, host, servletPath, parts);
         if (result != null) {
             deliver(response, result);
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobStatusAPIIntegrationTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobStatusAPIIntegrationTest.java
index f2f8061..6767eec 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobStatusAPIIntegrationTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/JobStatusAPIIntegrationTest.java
@@ -40,7 +40,7 @@
 
     @Test
     public void testNoRunningJobs() throws Exception {
-        Assert.assertEquals(0, countJobs("RUNNING"));
+        Assert.assertEquals(0, countJobs("STARTED"));
     }
 
     @Test
@@ -92,7 +92,7 @@
         JobId jId = executeTest(spec);
         // don't run for more than 100 s
         int maxLoops = 1000;
-        while (maxLoops > 0 && !"RUNNING".equals(getJobStatus(jId))) {
+        while (maxLoops > 0 && !"STARTED".equals(getJobStatus(jId))) {
             Thread.sleep(100);
             --maxLoops;
         }

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1550
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie0e58cac20c1976610f38796d06f0518a3174c50
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Till Westmann <ti...@apache.org>