You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mh...@apache.org on 2018/03/08 21:45:54 UTC
asterixdb git commit: [NO ISSUE][CLUS] Fail Tasks When NC Not Active
Repository: asterixdb
Updated Branches:
refs/heads/master c8ba268f7 -> 31589f960
[NO ISSUE][CLUS] Fail Tasks When NC Not Active
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Fail StartTasksWork if NC is not in active
state (i.e. hasn't completed startup tasks).
- Set NC state to active only after completing
startup tasks.
- Add test case.
Change-Id: I447d4f1e255211e9026b68eb9d931f25846ed153
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2460
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/31589f96
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/31589f96
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/31589f96
Branch: refs/heads/master
Commit: 31589f9604be0ed5026ceca696934b4a612d42ea
Parents: c8ba268
Author: Murtadha Hubail <mh...@apache.org>
Authored: Thu Mar 8 00:28:04 2018 +0300
Committer: Murtadha Hubail <mh...@apache.org>
Committed: Thu Mar 8 13:44:33 2018 -0800
----------------------------------------------------------------------
.../app/replication/NcLifecycleCoordinator.java | 3 +-
.../hyracks/bootstrap/NCApplication.java | 4 +-
.../apache/asterix/runtime/NcLifecycleTest.java | 64 ++++++++++++++++++++
.../apache/hyracks/api/client/NodeStatus.java | 7 ++-
.../hyracks/api/exceptions/ErrorCode.java | 1 +
.../src/main/resources/errormsg/en.properties | 1 +
.../control/nc/NodeControllerService.java | 6 +-
.../hyracks/control/nc/work/StartTasksWork.java | 5 ++
8 files changed, 83 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
index 844851a..5c497aa 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
@@ -146,7 +146,7 @@ public class NcLifecycleCoordinator implements INcLifecycleCoordinator {
return buildActiveNCRegTasks(isMetadataNode);
}
final List<INCLifecycleTask> tasks = new ArrayList<>();
- tasks.add(new UpdateNodeStatusTask(NodeStatus.ACTIVE));
+ tasks.add(new UpdateNodeStatusTask(NodeStatus.BOOTING));
if (state == SystemState.CORRUPTED) {
//need to perform local recovery for node partitions
LocalRecoveryTask rt = new LocalRecoveryTask(Arrays.asList(clusterManager.getNodePartitions(nodeId))
@@ -167,6 +167,7 @@ public class NcLifecycleCoordinator implements INcLifecycleCoordinator {
tasks.add(new BindMetadataNodeTask());
}
tasks.add(new ReportLocalCountersTask());
+ tasks.add(new UpdateNodeStatusTask(NodeStatus.ACTIVE));
return tasks;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index 0cd01ae..494198b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -89,6 +89,8 @@ public class NCApplication extends BaseNCApplication {
@Override
public void init(IServiceContext serviceCtx) throws Exception {
ncServiceCtx = (INCServiceContext) serviceCtx;
+ // set the node status initially to idle to indicate that it is pending booting
+ ((NodeControllerService) serviceCtx.getControllerService()).setNodeStatus(NodeStatus.IDLE);
ncServiceCtx.setThreadFactory(
new AsterixThreadFactory(ncServiceCtx.getThreadFactory(), ncServiceCtx.getLifeCycleComponentManager()));
}
@@ -253,7 +255,7 @@ public class NCApplication extends BaseNCApplication {
}
private boolean isPendingStartupTasks(NodeStatus nodeStatus, CcId primaryCc, CcId registeredCc) {
- return nodeStatus == NodeStatus.BOOTING && (primaryCc == null || primaryCc.equals(registeredCc));
+ return nodeStatus == NodeStatus.IDLE && (primaryCc == null || primaryCc.equals(registeredCc));
}
private SystemState getCurrentSystemState() {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NcLifecycleTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NcLifecycleTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NcLifecycleTest.java
new file mode 100644
index 0000000..df0740b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NcLifecycleTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime;
+
+import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
+import org.apache.asterix.common.TestDataUtil;
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.hyracks.api.client.NodeStatus;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class NcLifecycleTest {
+
+ protected static final String TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
+ private static final TestExecutor testExecutor = new TestExecutor();
+ private static final AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil();
+
+ @Before
+ public void setUp() throws Exception {
+ System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, TEST_CONFIG_FILE_NAME);
+ integrationUtil.init(true, TEST_CONFIG_FILE_NAME);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ integrationUtil.deinit(true);
+ }
+
+ @Test
+ public void nodeNotActiveTest() throws Exception {
+ final String datasetName = "ds";
+ TestDataUtil.createIdOnlyDataset(datasetName);
+ integrationUtil.ncs[0].setNodeStatus(NodeStatus.BOOTING);
+ boolean thrown = false;
+ try {
+ testExecutor.executeSqlppUpdateOrDdl("select count(*) from " + datasetName + ";",
+ TestCaseContext.OutputFormat.CLEAN_JSON);
+ } catch (Exception e) {
+ thrown = true;
+ Assert.assertTrue(e.getMessage().contains(integrationUtil.ncs[0].getId()));
+ }
+ Assert.assertTrue(thrown);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/NodeStatus.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/NodeStatus.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/NodeStatus.java
index 10a9a3c..076dd10 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/NodeStatus.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/NodeStatus.java
@@ -19,7 +19,8 @@
package org.apache.hyracks.api.client;
public enum NodeStatus {
- ACTIVE,
- BOOTING,
- DEAD
+ ACTIVE, // node is ready to process tasks
+ BOOTING, // node is executing startup tasks
+ DEAD,
+ IDLE
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
index 0dca782..0691005 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
@@ -147,6 +147,7 @@ public class ErrorCode {
public static final int CANNOT_CONTINUE_TEXT_SEARCH_BUFFER_MANAGER_IS_NULL = 111;
public static final int CANNOT_ADD_ELEMENT_TO_INVERTED_INDEX_SEARCH_RESULT = 112;
public static final int UNDEFINED_INVERTED_LIST_MERGE_TYPE = 113;
+ public static final int NODE_IS_NOT_ACTIVE = 114;
// Compilation error codes.
public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 10000;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
index 50e1ad4..8635efd 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
@@ -130,5 +130,6 @@
111 = To conduct an inverted-index search, the given buffer manager cannot be null.
112 = Cannot add an element to an inverted-index search result.
113 = Undefined inverted-list merge type: %1$s
+114 = Node (%1$s) is not active
10000 = The given rule collection %1$s is not an instance of the List class.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index b67bfac..2c73df1 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -170,7 +170,7 @@ public class NodeControllerService implements IControllerService {
private final Map<CcId, AtomicLong> maxJobIds = new ConcurrentHashMap<>();
- private NodeStatus status = NodeStatus.BOOTING;
+ private volatile NodeStatus status = NodeStatus.ACTIVE;
private NodeRegistration nodeRegistration;
@@ -633,11 +633,11 @@ public class NodeControllerService implements IControllerService {
return workQueue;
}
- public synchronized NodeStatus getNodeStatus() {
+ public NodeStatus getNodeStatus() {
return status;
}
- public synchronized void setNodeStatus(NodeStatus status) {
+ public void setNodeStatus(NodeStatus status) {
this.status = status;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java
index e229149..6a5785a 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.hyracks.api.application.INCServiceContext;
+import org.apache.hyracks.api.client.NodeStatus;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.comm.IPartitionCollector;
import org.apache.hyracks.api.comm.IPartitionWriterFactory;
@@ -45,6 +46,7 @@ import org.apache.hyracks.api.dataflow.connectors.IConnectorPolicy;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.deployment.DeploymentId;
+import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.job.ActivityCluster;
@@ -116,6 +118,9 @@ public class StartTasksWork extends AbstractWork {
ncs.updateMaxJobId(jobId);
NCServiceContext serviceCtx = ncs.getContext();
Joblet joblet = getOrCreateLocalJoblet(deploymentId, serviceCtx, acgBytes);
+ if (ncs.getNodeStatus() != NodeStatus.ACTIVE) {
+ throw HyracksException.create(ErrorCode.NODE_IS_NOT_ACTIVE, ncs.getId());
+ }
final ActivityClusterGraph acg = joblet.getActivityClusterGraph();
IRecordDescriptorProvider rdp = new IRecordDescriptorProvider() {
@Override