You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org> on 2017/08/27 17:15:46 UTC

Change in asterixdb[master]: [NO ISSUE][TEST][CLUS] Update cluster state on shutdown, tes...

Michael Blow has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/1978

Change subject: [NO ISSUE][TEST][CLUS] Update cluster state on shutdown, testfwk looping
......................................................................

[NO ISSUE][TEST][CLUS] Update cluster state on shutdown, testfwk looping

- Set ClusterState to SHUTTING_DOWN when the cluster is shutting down
- Add ability to TestExecutor to loop over test files based on iteration count
  or duration

Change-Id: I2bd67cb92a60d76eabe1e7649d16193dd72615dd
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ShutdownApiServlet.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
M asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
A asterixdb/asterix-app/src/test/resources/runtimets/queries/api/diagnostics_1/diagnostics_1.2.loop.cmd
R asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/Duration.java
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IClusterManagementWork.java
9 files changed, 155 insertions(+), 8 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/78/1978/1

diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
index 69e995b..8d355ef 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
@@ -29,6 +29,7 @@
 import org.apache.asterix.app.message.ExecuteStatementRequestMessage;
 import org.apache.asterix.app.message.ExecuteStatementResponseMessage;
 import org.apache.asterix.app.result.ResultReader;
+import org.apache.asterix.common.api.Duration;
 import org.apache.asterix.common.api.IApplicationContext;
 import org.apache.asterix.common.config.GlobalConfig;
 import org.apache.asterix.common.exceptions.ErrorCode;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index c630636..fa67190 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -27,6 +27,7 @@
 import java.util.logging.Logger;
 
 import org.apache.asterix.algebra.base.ILangExtension;
+import org.apache.asterix.common.api.Duration;
 import org.apache.asterix.common.api.IApplicationContext;
 import org.apache.asterix.common.api.IClusterManagementWork;
 import org.apache.asterix.common.config.GlobalConfig;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ShutdownApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ShutdownApiServlet.java
index 06e2383..38f6691 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ShutdownApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ShutdownApiServlet.java
@@ -19,6 +19,7 @@
 package org.apache.asterix.api.http.server;
 
 import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.common.api.IClusterManagementWork.ClusterState.SHUTTING_DOWN;
 
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -92,6 +93,8 @@
             jsonObject.set("cluster", clusterState);
             final PrintWriter writer = response.writer();
             writer.print(JSONUtil.convertNode(jsonObject));
+            // accept no further queries once this servlet returns
+            ClusterStateManager.INSTANCE.setState(SHUTTING_DOWN);
             writer.close();
         } catch (Exception e) {
             GlobalConfig.ASTERIX_LOGGER.log(Level.INFO, "Exception writing response", e);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
index b08c1e2..6d817b8 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
@@ -23,6 +23,7 @@
 import static org.apache.asterix.algebra.base.ILangExtension.Language.SQLPP;
 import static org.apache.asterix.api.http.server.ServletConstants.ASTERIX_APP_CONTEXT_INFO_ATTR;
 import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.common.api.IClusterManagementWork.ClusterState.SHUTTING_DOWN;
 
 import java.util.Arrays;
 import java.util.List;
@@ -187,12 +188,13 @@
 
     @Override
     public void stop() throws Exception {
-        if (appCtx != null) {
-            ((ActiveNotificationHandler) appCtx.getActiveNotificationHandler()).stop();
-        }
         if (LOGGER.isLoggable(Level.INFO)) {
             LOGGER.info("Stopping Asterix cluster controller");
         }
+        ClusterStateManager.INSTANCE.setState(SHUTTING_DOWN);
+        if (appCtx != null) {
+            ((ActiveNotificationHandler) appCtx.getActiveNotificationHandler()).stop();
+        }
         AsterixStateProxy.unregisterRemoteObject();
         webManager.stop();
     }
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java
index 12c61d6..f2fb580 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java
@@ -18,7 +18,7 @@
  */
 package org.apache.asterix.runtime;
 
-import org.apache.asterix.api.http.server.Duration;
+import org.apache.asterix.common.api.Duration;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.junit.Assert;
 import org.junit.Test;
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index e07ec72..b7e4935 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -43,6 +43,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.ListIterator;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
@@ -56,6 +57,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.asterix.common.api.Duration;
 import org.apache.asterix.common.config.GlobalConfig;
 import org.apache.asterix.common.utils.Servlets;
 import org.apache.asterix.test.server.ITestServer;
@@ -131,6 +133,7 @@
     protected final List<InetSocketAddress> endpoints;
     protected int endpointSelector;
     protected ITestLibrarian librarian;
+    private Map<File, TestLoop> testLoops = new HashMap<>();
 
     public TestExecutor() {
         this(Inet4Address.getLoopbackAddress().getHostAddress(), 19002);
@@ -1080,6 +1083,59 @@
                     killNC(nodeId, cUnit);
                 }
                 break;
+            case "loop":
+                TestLoop testLoop = testLoops.get(testFile);
+                if (testLoop == null) {
+                    lines = stripAllComments(statement).trim().split("\n");
+                    String target = null;
+                    int count = -1;
+                    long durationSecs = -1;
+                    for (String line : lines) {
+                        command = line.trim().split(" ");
+                        switch (command[0]) {
+                            case "target":
+                                if (target != null) {
+                                    throw new IllegalStateException("duplicate target");
+                                }
+                                target = command[1];
+                                break;
+                            case "count":
+                                if (count != -1) {
+                                    throw new IllegalStateException("duplicate count");
+                                }
+                                count = Integer.parseInt(command[1]);
+                                break;
+                            case "duration":
+                                if (durationSecs != -1) {
+                                    throw new IllegalStateException("duplicate duration");
+                                }
+                                long duration = Duration.parseDurationStringToNanos(command[1]);
+                                durationSecs = TimeUnit.NANOSECONDS.toSeconds(duration);
+                                if (durationSecs < 1) {
+                                    throw new IllegalArgumentException("duration cannot be shorter than 1s");
+                                } else if (TimeUnit.SECONDS.toDays(durationSecs) > 1) {
+                                    throw new IllegalArgumentException("duration cannot be exceed 1d");
+                                }
+                                break;
+                            default:
+                                throw new IllegalArgumentException("unknown directive: " + command[0]);
+                        }
+                    }
+                    if (target == null || (count == -1 && durationSecs == -1) || (count != -1 && durationSecs != -1)) {
+                        throw new IllegalStateException("Must specify 'target' and exactly one of 'count', 'duration'");
+                    }
+                    if (count != -1) {
+                        testLoop = TestLoop.createLoop(target, count);
+                    } else {
+                        testLoop = TestLoop.createLoop(target, durationSecs, TimeUnit.SECONDS);
+                    }
+                    testLoops.put(testFile, testLoop);
+                }
+                testLoop.executeLoop();
+                // we only reach here if the loop is over
+                testLoops.remove(testFile);
+                break;
+
             default:
                 throw new IllegalArgumentException("No statements of type " + ctx.getType());
         }
@@ -1089,7 +1145,7 @@
             String reqType, File testFile, File expectedResultFile, File actualResultFile, MutableInt queryCount,
             int numResultFiles, String extension, ComparisonEnum compare) throws Exception {
         String handleVar = getHandleVariable(statement);
-        final String trimmedPathAndQuery = stripLineComments(stripJavaComments(statement)).trim();
+        final String trimmedPathAndQuery = stripAllComments(statement).trim();
         final String variablesReplaced = replaceVarRef(trimmedPathAndQuery, variableCtx);
         final List<Parameter> params = extractParameters(statement);
         final Predicate<Integer> statusCodePredicate = extractStatusCodePredicate(statement);
@@ -1361,13 +1417,27 @@
             Map<String, Object> variableCtx = new HashMap<>();
             List<TestFileContext> testFileCtxs = testCaseCtx.getTestFiles(cUnit);
             List<TestFileContext> expectedResultFileCtxs = testCaseCtx.getExpectedResultFiles(cUnit);
-            for (TestFileContext ctx : testFileCtxs) {
+            int[] savedQueryCounts = new int[testFileCtxs.size()];
+            int offset = numOfFiles;
+            for (ListIterator<TestFileContext> iter = testFileCtxs.listIterator(); iter.hasNext();) {
+                TestFileContext ctx = iter.next();
+                savedQueryCounts[numOfFiles + offset] = queryCount.getValue();
                 numOfFiles++;
                 final File testFile = ctx.getFile();
                 final String statement = readTestFile(testFile);
                 try {
                     executeTestFile(testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit, queryCount,
                             expectedResultFileCtxs, testFile, actualPath);
+                } catch (TestLoop loop) {
+                    // rewind the iterator until we find our target
+                    while (!ctx.getFile().getName().equals(loop.getTarget())) {
+                        if (!iter.hasPrevious()) {
+                            throw new IllegalStateException("unable to find loop target '" + loop.getTarget() + "'!");
+                        }
+                        ctx = iter.previous();
+                        numOfFiles--;
+                        queryCount.setValue(savedQueryCounts[numOfFiles + offset]);
+                    }
                 } catch (Exception e) {
                     System.err.println("testFile " + testFile.toString() + " raised an exception: " + e);
                     numOfErrors++;
@@ -1440,6 +1510,10 @@
     public static String stripLineComments(String text) {
         final String s = SHELL_LINE_COMMENT_PATTERN.matcher(text).replaceAll("");
         return JAVA_LINE_COMMENT_PATTERN.matcher(s).replaceAll("");
+    }
+
+    public static String stripAllComments(String statement) {
+        return stripLineComments(stripJavaComments(statement));
     }
 
     public void cleanup(String testCase, List<String> badtestcases) throws Exception {
@@ -1531,4 +1605,48 @@
         LOGGER.info("Cluster state now " + desiredState);
     }
 
+    abstract static class TestLoop extends Exception {
+
+        private final String target;
+
+        TestLoop(String target) {
+            this.target = target;
+        }
+
+        static TestLoop createLoop(String target, final int count) {
+            LOGGER.info("Starting loop '" + count + " times back to '" + target + "'...");
+            return new TestLoop(target) {
+                int remainingLoops = count;
+
+                @Override
+                void executeLoop() throws TestLoop {
+                    if (remainingLoops-- > 0) {
+                        throw this;
+                    }
+                    LOGGER.info("Loop to '" + target + "' complete!");
+                }
+            };
+        }
+
+        static TestLoop createLoop(String target, long duration, TimeUnit unit) {
+            LOGGER.info("Starting loop for " + unit.toSeconds(duration) + "s back to '" + target + "'...");
+            return new TestLoop(target) {
+                long endTime = unit.toMillis(duration) + System.currentTimeMillis();
+
+                @Override
+                void executeLoop() throws TestLoop {
+                    if (System.currentTimeMillis() < endTime) {
+                        throw this;
+                    }
+                    LOGGER.info("Loop to '" + target + "' complete!");
+                }
+            };
+        }
+
+        abstract void executeLoop() throws TestLoop;
+
+        public String getTarget() {
+            return target;
+        }
+    }
 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/diagnostics_1/diagnostics_1.2.loop.cmd b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/diagnostics_1/diagnostics_1.2.loop.cmd
new file mode 100644
index 0000000..2660b46
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/diagnostics_1/diagnostics_1.2.loop.cmd
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+target diagnostics_1.1.get.http
+count 2
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/Duration.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/Duration.java
similarity index 99%
rename from asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/Duration.java
rename to asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/Duration.java
index bdda750..4338222 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/Duration.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/Duration.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.api.http.server;
+package org.apache.asterix.common.api;
 
 import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.asterix.common.exceptions.RuntimeDataException;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IClusterManagementWork.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IClusterManagementWork.java
index 323df65..bdbf4a5 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IClusterManagementWork.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IClusterManagementWork.java
@@ -30,7 +30,8 @@
         PENDING,
         ACTIVE,
         UNUSABLE,
-        REBALANCING
+        REBALANCING,
+        SHUTTING_DOWN
     }
 
     public WorkType getClusterManagementWorkType();

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1978
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I2bd67cb92a60d76eabe1e7649d16193dd72615dd
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>