You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org> on 2017/08/17 17:47:43 UTC

Change in asterixdb[master]: [NO ISSUE][HYR][*DB][HTTP][TEST] Minor refactoring / test fu...

Michael Blow has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/1948

Change subject: [NO ISSUE][HYR][*DB][HTTP][TEST] Minor refactoring / test functionality
......................................................................

[NO ISSUE][HYR][*DB][HTTP][TEST] Minor refactoring / test functionality

Adds ability to indicate which status code(s) to accept for an HTTP
request, instead of just 200.

Change-Id: I1f134d7874cc7de712e60911e89dac446c5c2cfd
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
M hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
3 files changed, 76 insertions(+), 49 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/48/1948/1

diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
index 82e8f7a..b24a9a1 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
@@ -26,7 +26,6 @@
 import java.util.function.Predicate;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.regex.Pattern;
 
 import org.apache.asterix.runtime.utils.CcApplicationContext;
 import org.apache.asterix.runtime.utils.ClusterStateManager;
@@ -50,7 +49,6 @@
 public class ClusterApiServlet extends AbstractServlet {
 
     private static final Logger LOGGER = Logger.getLogger(ClusterApiServlet.class.getName());
-    private static final Pattern PARENT_DIR = Pattern.compile("/[^./]+/\\.\\./");
     protected static final String NODE_ID_KEY = "node_id";
     protected static final String CONFIG_URI_KEY = "configUri";
     protected static final String STATS_URI_KEY = "statsUri";
@@ -104,15 +102,8 @@
                 Section.COMMON, getConfigSelector()));
 
         ArrayNode ncs = (ArrayNode) json.get("ncs");
-        final StringBuilder requestURL = new StringBuilder("http://");
-        requestURL.append(request.getHeader(HttpHeaderNames.HOST));
-        requestURL.append(request.getHttpRequest().uri());
-        if (requestURL.charAt(requestURL.length() - 1) != '/') {
-            requestURL.append('/');
-        }
-        requestURL.append(pathToNode);
-        String clusterURL = canonicalize(requestURL);
-        String adminURL = canonicalize(clusterURL + "../");
+        String clusterURL = resolveClusterUrl(request, pathToNode);
+        String adminURL = HttpUtil.canonicalize(clusterURL + "../");
         String nodeURL = clusterURL + "node/";
         for (int i = 0; i < ncs.size(); i++) {
             ObjectNode nc = (ObjectNode) ncs.get(i);
@@ -137,18 +128,20 @@
         return json;
     }
 
+    protected String resolveClusterUrl(IServletRequest request, String pathToNode) {
+        final StringBuilder requestURL = new StringBuilder("http://");
+        requestURL.append(request.getHeader(HttpHeaderNames.HOST));
+        requestURL.append(request.getHttpRequest().uri());
+        if (requestURL.charAt(requestURL.length() - 1) != '/') {
+            requestURL.append('/');
+        }
+        requestURL.append(pathToNode);
+        return HttpUtil.canonicalize(requestURL);
+    }
+
     protected Predicate<IOption> getConfigSelector() {
         return option -> !option.hidden() && option != ControllerConfig.Option.CONFIG_FILE
                 && option != ControllerConfig.Option.CONFIG_FILE_URL;
     }
 
-    private String canonicalize(CharSequence requestURL) {
-        String clusterURL = "";
-        String newClusterURL = requestURL.toString();
-        while (!clusterURL.equals(newClusterURL)) {
-            clusterURL = newClusterURL;
-            newClusterURL = PARENT_DIR.matcher(clusterURL).replaceAll("/");
-        }
-        return clusterURL;
-    }
 }
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index a96adc0..e07ec72 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -103,17 +103,18 @@
     // see
     // https://stackoverflow.com/questions/417142/what-is-the-maximum-length-of-a-url-in-different-browsers/417184
     private static final long MAX_URL_LENGTH = 2000l;
-    private static final Pattern JAVA_BLOCK_COMMENT_PATTERN =
-            Pattern.compile("/\\*.*\\*/", Pattern.MULTILINE | Pattern.DOTALL);
+    private static final Pattern JAVA_BLOCK_COMMENT_PATTERN = Pattern.compile("/\\*.*\\*/",
+            Pattern.MULTILINE | Pattern.DOTALL);
     private static final Pattern JAVA_LINE_COMMENT_PATTERN = Pattern.compile("^//.*$", Pattern.MULTILINE);
     private static final Pattern SHELL_LINE_COMMENT_PATTERN = Pattern.compile("^#.*$", Pattern.MULTILINE);
     private static final Pattern REGEX_LINES_PATTERN = Pattern.compile("^(-)?/(.*)/([im]*)$");
-    private static final Pattern POLL_TIMEOUT_PATTERN =
-            Pattern.compile("polltimeoutsecs=(\\d+)(\\D|$)", Pattern.MULTILINE);
+    private static final Pattern POLL_TIMEOUT_PATTERN = Pattern.compile("polltimeoutsecs=(\\d+)(\\D|$)",
+            Pattern.MULTILINE);
     private static final Pattern POLL_DELAY_PATTERN = Pattern.compile("polldelaysecs=(\\d+)(\\D|$)", Pattern.MULTILINE);
     private static final Pattern HANDLE_VARIABLE_PATTERN = Pattern.compile("handlevariable=(\\w+)");
     private static final Pattern VARIABLE_REF_PATTERN = Pattern.compile("\\$(\\w+)");
     private static final Pattern HTTP_PARAM_PATTERN = Pattern.compile("param (\\w+)=(.*)", Pattern.MULTILINE);
+    private static final Pattern HTTP_STATUSCODE_PATTERN = Pattern.compile("statuscode (.*)", Pattern.MULTILINE);
 
     public static final int TRUNCATE_THRESHOLD = 16384;
 
@@ -168,10 +169,10 @@
     public void runScriptAndCompareWithResult(File scriptFile, PrintWriter print, File expectedFile, File actualFile,
             ComparisonEnum compare) throws Exception {
         System.err.println("Expected results file: " + expectedFile.toString());
-        BufferedReader readerExpected =
-                new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), "UTF-8"));
-        BufferedReader readerActual =
-                new BufferedReader(new InputStreamReader(new FileInputStream(actualFile), "UTF-8"));
+        BufferedReader readerExpected = new BufferedReader(
+                new InputStreamReader(new FileInputStream(expectedFile), "UTF-8"));
+        BufferedReader readerActual = new BufferedReader(
+                new InputStreamReader(new FileInputStream(actualFile), "UTF-8"));
         boolean regex = false;
         try {
             if (ComparisonEnum.BINARY.equals(compare)) {
@@ -354,10 +355,10 @@
     public void runScriptAndCompareWithResultRegex(File scriptFile, File expectedFile, File actualFile)
             throws Exception {
         String lineExpected, lineActual;
-        try (BufferedReader readerExpected =
-                new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), "UTF-8"));
-                BufferedReader readerActual =
-                        new BufferedReader(new InputStreamReader(new FileInputStream(actualFile), "UTF-8"))) {
+        try (BufferedReader readerExpected = new BufferedReader(
+                new InputStreamReader(new FileInputStream(expectedFile), "UTF-8"));
+                BufferedReader readerActual = new BufferedReader(
+                        new InputStreamReader(new FileInputStream(actualFile), "UTF-8"))) {
             StringBuilder actual = new StringBuilder();
             while ((lineActual = readerActual.readLine()) != null) {
                 actual.append(lineActual).append('\n');
@@ -697,8 +698,8 @@
     // Insert and Delete statements are executed here
     public void executeUpdate(String str, URI uri) throws Exception {
         // Create a method instance.
-        HttpUriRequest request =
-                RequestBuilder.post(uri).setEntity(new StringEntity(str, StandardCharsets.UTF_8)).build();
+        HttpUriRequest request = RequestBuilder.post(uri).setEntity(new StringEntity(str, StandardCharsets.UTF_8))
+                .build();
 
         // Execute the method.
         executeAndCheckHttpRequest(request);
@@ -708,10 +709,10 @@
     public InputStream executeAnyAQLAsync(String statement, boolean defer, OutputFormat fmt, URI uri,
             Map<String, Object> variableCtx) throws Exception {
         // Create a method instance.
-        HttpUriRequest request =
-                RequestBuilder.post(uri).addParameter("mode", defer ? "asynchronous-deferred" : "asynchronous")
-                        .setEntity(new StringEntity(statement, StandardCharsets.UTF_8))
-                        .setHeader("Accept", fmt.mimeType()).build();
+        HttpUriRequest request = RequestBuilder.post(uri)
+                .addParameter("mode", defer ? "asynchronous-deferred" : "asynchronous")
+                .setEntity(new StringEntity(statement, StandardCharsets.UTF_8)).setHeader("Accept", fmt.mimeType())
+                .build();
 
         String handleVar = getHandleVariable(statement);
 
@@ -737,8 +738,8 @@
     // create function statement
     public void executeDDL(String str, URI uri) throws Exception {
         // Create a method instance.
-        HttpUriRequest request =
-                RequestBuilder.post(uri).setEntity(new StringEntity(str, StandardCharsets.UTF_8)).build();
+        HttpUriRequest request = RequestBuilder.post(uri).setEntity(new StringEntity(str, StandardCharsets.UTF_8))
+                .build();
 
         // Execute the method.
         executeAndCheckHttpRequest(request);
@@ -748,8 +749,8 @@
     // and returns the contents as a string
     // This string is later passed to REST API for execution.
     public String readTestFile(File testFile) throws Exception {
-        BufferedReader reader =
-                new BufferedReader(new InputStreamReader(new FileInputStream(testFile), StandardCharsets.UTF_8));
+        BufferedReader reader = new BufferedReader(
+                new InputStreamReader(new FileInputStream(testFile), StandardCharsets.UTF_8));
         String line;
         StringBuilder stringBuilder = new StringBuilder();
         String ls = System.getProperty("line.separator");
@@ -804,8 +805,8 @@
 
     private static String getProcessOutput(Process p) throws Exception {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        Future<Integer> future =
-                Executors.newSingleThreadExecutor().submit(() -> IOUtils.copy(p.getInputStream(), new OutputStream() {
+        Future<Integer> future = Executors.newSingleThreadExecutor()
+                .submit(() -> IOUtils.copy(p.getInputStream(), new OutputStream() {
                     @Override
                     public void write(int b) throws IOException {
                         baos.write(b);
@@ -1091,11 +1092,12 @@
         final String trimmedPathAndQuery = stripLineComments(stripJavaComments(statement)).trim();
         final String variablesReplaced = replaceVarRef(trimmedPathAndQuery, variableCtx);
         final List<Parameter> params = extractParameters(statement);
+        final Predicate<Integer> statusCodePredicate = extractStatusCodePredicate(statement);
         InputStream resultStream;
         if ("http".equals(extension)) {
-            resultStream = executeHttp(reqType, variablesReplaced, fmt, params);
+            resultStream = executeHttp(reqType, variablesReplaced, fmt, params, statusCodePredicate);
         } else if ("uri".equals(extension)) {
-            resultStream = executeURI(reqType, URI.create(variablesReplaced), fmt, params);
+            resultStream = executeURI(reqType, URI.create(variablesReplaced), fmt, params, statusCodePredicate);
         } else {
             throw new IllegalArgumentException("Unexpected format for method " + reqType + ": " + extension);
         }
@@ -1287,17 +1289,35 @@
         return params;
     }
 
-    protected InputStream executeHttp(String ctxType, String endpoint, OutputFormat fmt, List<Parameter> params)
-            throws Exception {
+    protected static Predicate<Integer> extractStatusCodePredicate(String statement) {
+        List<Integer> codes = new ArrayList<>();
+        final Matcher m = HTTP_STATUSCODE_PATTERN.matcher(statement);
+        while (m.find()) {
+            codes.add(Integer.parseInt(m.group(1)));
+        }
+        if (codes.isEmpty()) {
+            return code -> code == HttpStatus.SC_OK;
+        } else {
+            return codes::contains;
+        }
+    }
+
+    protected InputStream executeHttp(String ctxType, String endpoint, OutputFormat fmt, List<Parameter> params,
+            Predicate<Integer> statusCodePredicate) throws Exception {
         String[] split = endpoint.split("\\?");
         URI uri = createEndpointURI(split[0], split.length > 1 ? split[1] : null);
-        return executeURI(ctxType, uri, fmt, params);
+        return executeURI(ctxType, uri, fmt, params, statusCodePredicate);
     }
 
     private InputStream executeURI(String ctxType, URI uri, OutputFormat fmt, List<Parameter> params) throws Exception {
         return executeJSON(fmt, ctxType.toUpperCase(), uri, params);
     }
 
+    private InputStream executeURI(String ctxType, URI uri, OutputFormat fmt, List<Parameter> params,
+            Predicate<Integer> responseCodeValidator) throws Exception {
+        return executeJSON(fmt, ctxType.toUpperCase(), uri, params, responseCodeValidator);
+    }
+
     private void killNC(String nodeId, CompilationUnit cUnit) throws Exception {
         //get node process id
         OutputFormat fmt = OutputFormat.forCompilationUnit(cUnit);
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
index 2babc73..fa3cc57 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
@@ -22,6 +22,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;
 
 import org.apache.hyracks.http.api.IServletRequest;
 import org.apache.hyracks.http.api.IServletResponse;
@@ -33,6 +34,8 @@
 import io.netty.handler.codec.http.HttpRequest;
 
 public class HttpUtil {
+
+    private static final Pattern PARENT_DIR = Pattern.compile("/[^./]+/\\.\\./");
 
     private HttpUtil() {
     }
@@ -131,4 +134,15 @@
                 return null;
         }
     }
+
+    public static String canonicalize(CharSequence requestURL) {
+        String clusterURL = "";
+        String newClusterURL = requestURL.toString();
+        while (!clusterURL.equals(newClusterURL)) {
+            clusterURL = newClusterURL;
+            newClusterURL = PARENT_DIR.matcher(clusterURL).replaceAll("/");
+        }
+        return clusterURL;
+    }
+
 }

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1948
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I1f134d7874cc7de712e60911e89dac446c5c2cfd
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>