You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/11/01 12:49:14 UTC
[02/14] incubator-slider git commit: SLIDER-82 SLIDER-947 build node
map from yarn update reports; serve via REST/IPC
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
index c408ed2..7425c2d 100644
--- a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
+++ b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
@@ -82,7 +82,8 @@ public class RestTypeMarshalling {
if (wire.hasFailureMessage()) {
info.failureMessage = wire.getFailureMessage();
}
-
+ info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount();
+ info.pendingAntiAffineRequest = info.pendingAntiAffineRequestCount > 0;
return info;
}
@@ -132,11 +133,11 @@ public class RestTypeMarshalling {
if (info.containers != null) {
builder.addAllContainers(info.containers);
}
+ builder.setPendingAntiAffineRequestCount(info.pendingAntiAffineRequestCount);
return builder.build();
}
- public static ContainerInformation
- unmarshall(Messages.ContainerInformationProto wire) {
+ public static ContainerInformation unmarshall(Messages.ContainerInformationProto wire) {
ContainerInformation info = new ContainerInformation();
info.containerId = wire.getContainerId();
info.component = wire.getComponent();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
index 52f6c08..3b4b8bd 100644
--- a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
+++ b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
@@ -19,6 +19,7 @@
package org.apache.slider.api.types;
import org.apache.slider.api.StatusKeys;
+import org.apache.slider.server.appmaster.state.RoleStatus;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
@@ -35,13 +36,15 @@ import java.util.Map;
* <p>
* This means that if any fields are added here. they must be added to
* <code>src/main/proto/SliderClusterMessages.proto</code> and
- * the probuf structures rebuilt.
+ * the protobuf structures rebuilt via a {@code mvn generate-sources -Pcompile-protobuf}
+ *
+ * See also {@link RoleStatus#serialize()}
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class ComponentInformation {
-
+
public String name;
public int priority;
public int desired, actual, releasing;
@@ -49,6 +52,9 @@ public class ComponentInformation {
public int requested;
public int failed, started, startFailed, completed, totalRequested;
public int nodeFailed, failedRecently, preempted;
+ public boolean pendingAntiAffineRequest;
+ public int pendingAntiAffineRequestCount;
+
public String failureMessage;
public List<String> containers;
@@ -59,16 +65,17 @@ public class ComponentInformation {
public Map<String, Integer> buildStatistics() {
Map<String, Integer> stats = new HashMap<>();
stats.put(StatusKeys.STATISTICS_CONTAINERS_ACTIVE_REQUESTS, requested);
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_ANTI_AFFINE_PENDING, pendingAntiAffineRequestCount);
stats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED, completed);
stats.put(StatusKeys.STATISTICS_CONTAINERS_DESIRED, desired);
stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED, failed);
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_NODE, nodeFailed);
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_RECENTLY, failedRecently);
stats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE, actual);
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_PREEMPTED, preempted);
stats.put(StatusKeys.STATISTICS_CONTAINERS_REQUESTED, totalRequested);
stats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED, started);
stats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED, startFailed);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_RECENTLY, failedRecently);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_NODE, nodeFailed);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_PREEMPTED, preempted);
return stats;
}
@@ -76,22 +83,23 @@ public class ComponentInformation {
public String toString() {
final StringBuilder sb =
new StringBuilder("ComponentInformation{");
- sb.append("failureMessage='").append(failureMessage).append('\'');
- sb.append(", totalRequested=").append(totalRequested);
+ sb.append(", name='").append(name).append('\'');
+ sb.append(", actual=").append(actual);
+ sb.append(", anti-affine pending").append(pendingAntiAffineRequestCount);
sb.append(", completed=").append(completed);
- sb.append(", startFailed=").append(startFailed);
- sb.append(", started=").append(started);
+ sb.append(", desired=").append(desired);
sb.append(", failed=").append(failed);
- sb.append(", requested=").append(requested);
+ sb.append(", failureMessage='").append(failureMessage).append('\'');
sb.append(", placementPolicy=").append(placementPolicy);
- sb.append(", releasing=").append(releasing);
- sb.append(", actual=").append(actual);
- sb.append(", desired=").append(desired);
sb.append(", priority=").append(priority);
- sb.append(", name='").append(name).append('\'');
- sb.append(", container count='")
- .append(containers== null ? 0: containers.size())
- .append('\'');
+ sb.append(", releasing=").append(releasing);
+ sb.append(", requested=").append(requested);
+ sb.append(", started=").append(started);
+ sb.append(", startFailed=").append(startFailed);
+ sb.append(", totalRequested=").append(totalRequested);
+ sb.append("container count='")
+ .append(containers == null ? 0 : containers.size())
+ .append('\'');
sb.append('}');
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java
new file mode 100644
index 0000000..33a875e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ * Serialized node entry information. Must be kept in sync with the protobuf equivalent.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class NodeEntryInformation {
+
+ public int priority;
+
+ /**
+ * instance explicitly requested on this node: it's OK if an allocation
+ * comes in that has not been (and when that happens, this count should
+ * not drop).
+ */
+ public int requested;
+
+ /** number of starting instances */
+ public int starting;
+
+ /** incrementing counter of instances that failed to start */
+ public int startFailed;
+
+ /** incrementing counter of instances that failed */
+ public int failed;
+
+ /**
+ * Counter of "failed recently" events. These are all failures
+ * which have happened since it was last reset.
+ */
+ public int failedRecently;
+
+ /** incrementing counter of instances that have been pre-empted. */
+ public int preempted;
+
+ /**
+ * Number of live nodes.
+ */
+ public int live;
+
+ /** number of containers being released off this node */
+ public int releasing;
+
+ /** timestamp of last use */
+ public long lastUsed;
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java
new file mode 100644
index 0000000..842db14
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Serialized node information. Must be kept in sync with the protobuf equivalent.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class NodeInformation {
+
+ public String hostname;
+ public String state;
+ public String httpAddress;
+ public String rackName;
+ public String labels;
+ public String healthReport;
+ public long lastUpdated;
+ public List<NodeEntryInformation> entries = new ArrayList<>();
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java
deleted file mode 100644
index 32684c6..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import org.apache.slider.common.SliderExitCodes;
-import org.apache.slider.core.main.LauncherExitCodes;
-
-public class AMUtils {
- /**
- * Map an exit code from a process
- * @param exitCode
- * @return an exit code
- */
- public static int mapProcessExitCodeToYarnExitCode(int exitCode) {
- switch (exitCode) {
- case LauncherExitCodes.EXIT_SUCCESS:
- return LauncherExitCodes.EXIT_SUCCESS;
- //remap from a planned shutdown to a failure
- case LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN:
- return SliderExitCodes.EXIT_SUCCESS;
- default:
- return exitCode;
- }
- }
-
- public static boolean isMappedExitAFailure(int mappedExitCode) {
- return mappedExitCode != 0;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
new file mode 100644
index 0000000..50b5dad
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.slider.server.appmaster.state.AbstractClusterServices;
+
+public class ProtobufClusterServices extends AbstractClusterServices {
+
+ public Resource newResource() {
+ return Records.newRecord(Resource.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufRecordFactory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufRecordFactory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufRecordFactory.java
deleted file mode 100644
index d7f79f1..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufRecordFactory.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.slider.server.appmaster.state.AbstractRecordFactory;
-
-public class ProtobufRecordFactory extends AbstractRecordFactory {
- public Resource newResource() {
- return Records.newRecord(Resource.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 019ec71..0e7e295 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -291,7 +291,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* live on, etc.
*/
private final AppState appState =
- new AppState(new ProtobufRecordFactory(), metricsAndMonitoring);
+ new AppState(new ProtobufClusterServices(), metricsAndMonitoring);
/**
* App state for external objects. This is almost entirely
@@ -301,7 +301,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
private final ProviderAppState stateForProviders =
new ProviderAppState("undefined", appState);
-
/**
* model the state using locks and conditions
*/
@@ -1000,7 +999,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
.start(webApp);
WebAppService<SliderAMWebApp> webAppService =
- new WebAppService<SliderAMWebApp>("slider", webApp);
+ new WebAppService<>("slider", webApp);
deployChildService(webAppService);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
index 5b96256..8de27e7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
@@ -24,7 +24,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.CompositeService;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -49,8 +51,14 @@ public class MetricsAndMonitoring extends CompositeService {
final HealthCheckRegistry health = new HealthCheckRegistry();
private final Map<String, MeterAndCounter> meterAndCounterMap
- = new ConcurrentHashMap<String, MeterAndCounter>();
-
+ = new ConcurrentHashMap<>();
+
+ /**
+ * List of recorded events
+ */
+ private final List<RecordedEvent> eventHistory = new ArrayList<>(100);
+
+ public static final int EVENT_LIMIT = 1000;
public MetricRegistry getMetrics() {
return metrics;
@@ -100,4 +108,23 @@ public class MetricsAndMonitoring extends CompositeService {
meter.mark();
}
+ /**
+ * Add an event (synchronized)
+ * @param event event
+ */
+ public synchronized void noteEvent(RecordedEvent event) {
+ if (eventHistory.size() > EVENT_LIMIT) {
+ eventHistory.remove(0);
+ }
+ eventHistory.add(event);
+ }
+
+ /**
+ * Clone the event history; blocks for the duration of the copy operation.
+ * @return a new list
+ */
+ public synchronized List<RecordedEvent> cloneEventHistory() {
+ return new ArrayList<>(eventHistory);
+ }
}
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java
new file mode 100644
index 0000000..d48d337
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.text.DateFormat;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class RecordedEvent {
+ private static final DateFormat dateFormat = DateFormat.getDateInstance();
+ public long id;
+ public String name;
+ public long timestamp;
+ public String time;
+ public String category;
+ public String host;
+ public int role;
+ public String text;
+
+ public RecordedEvent() {
+ }
+
+ /**
+ * Create an event. The timestamp is also converted to a time string
+ * @param id id counter
+ * @param name event name
+ * @param timestamp timestamp. If non-zero, is used to build the {@code time} text field.
+ * @param category even category
+ * @param text arbitrary text
+ */
+ public RecordedEvent(long id, String name, long timestamp, String category, String text) {
+ this.id = id;
+ this.name = name;
+ this.timestamp = timestamp;
+ this.time = timestamp > 0 ? dateFormat.format(timestamp) : "";
+ this.category = category;
+ this.text = text;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
index 594ee47..b7794ed 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
@@ -25,6 +25,11 @@ import org.apache.hadoop.yarn.client.api.AMRMClient;
import java.util.List;
public interface RMOperationHandlerActions {
+
+ /**
+ * Release an assigned container
+ * @param containerId container
+ */
void releaseAssignedContainer(ContainerId containerId);
/**
@@ -38,7 +43,7 @@ public interface RMOperationHandlerActions {
* @param request request to cancel
*/
void cancelSingleRequest(AMRMClient.ContainerRequest request);
-
+
/**
* Remove a container request
* @param priority1 priority to remove at
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java
new file mode 100644
index 0000000..27e25f9
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+
+/**
+ * Cluster services offered by the YARN infrastructure.
+ */
+public abstract class AbstractClusterServices {
+ /**
+ * Create a resource for requests
+ * @return a resource which can be built up.
+ */
+ public abstract Resource newResource();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRecordFactory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRecordFactory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRecordFactory.java
deleted file mode 100644
index e9655f0..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRecordFactory.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-
-/**
- * Factory supplying records created by the App state; entry point
- * for mock code.
- */
-public abstract class AbstractRecordFactory {
-
- public abstract Resource newResource();
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 6990a24..0f77824 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -115,7 +115,7 @@ public class AppState {
protected static final Logger log =
LoggerFactory.getLogger(AppState.class);
- private final AbstractRecordFactory recordFactory;
+ private final AbstractClusterServices recordFactory;
private final MetricsAndMonitoring metricsAndMonitoring;
@@ -179,13 +179,13 @@ public class AppState {
private ClusterDescription clusterStatusTemplate = new ClusterDescription();
private final Map<Integer, RoleStatus> roleStatusMap =
- new ConcurrentHashMap<Integer, RoleStatus>();
+ new ConcurrentHashMap<>();
private final Map<String, ProviderRole> roles =
- new ConcurrentHashMap<String, ProviderRole>();
+ new ConcurrentHashMap<>();
private final Map<Integer, ProviderRole> rolePriorityMap =
- new ConcurrentHashMap<Integer, ProviderRole>();
+ new ConcurrentHashMap<>();
/**
* The master node.
@@ -265,7 +265,7 @@ public class AppState {
* Nodes that came assigned to a role above that
* which were asked for -this appears to happen
*/
- private final Set<ContainerId> surplusNodes = new HashSet<ContainerId>();
+ private final Set<ContainerId> surplusNodes = new HashSet<>();
/**
* Map of containerID to cluster nodes, for status reports.
@@ -308,7 +308,7 @@ public class AppState {
* @param recordFactory factory for YARN records
* @param metricsAndMonitoring metrics and monitoring services
*/
- public AppState(AbstractRecordFactory recordFactory,
+ public AppState(AbstractClusterServices recordFactory,
MetricsAndMonitoring metricsAndMonitoring) {
this.recordFactory = recordFactory;
this.metricsAndMonitoring = metricsAndMonitoring;
@@ -1439,6 +1439,10 @@ public class AppState {
}
}
+ /**
+ * Handle node update from the RM. This syncs up the node map with the RM's view
+ * @param updatedNodes updated nodes
+ */
public synchronized void onNodesUpdated(List<NodeReport> updatedNodes) {
roleHistory.onNodesUpdated(updatedNodes);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
index 0f46054..8ff0895 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
@@ -18,23 +18,24 @@
package org.apache.slider.server.appmaster.state;
+import org.apache.slider.api.types.NodeEntryInformation;
+
/**
* Information about the state of a role on a specific node instance.
* No fields are synchronized; sync on the instance to work with it
- <p>
- The two fields `releasing` and `requested` are used to track the ongoing
- state of YARN requests; they do not need to be persisted across stop/start
- cycles. They may be relevant across AM restart, but without other data
- structures in the AM, not enough to track what the AM was up to before
- it was restarted. The strategy will be to ignore unexpected allocation
- responses (which may come from pre-restart) requests, while treating
- unexpected container release responses as failures.
- <p>
- The `active` counter is only decremented after a container release response
- has been received.
- <p>
-
- Accesses are synchronized.
+ * <p>
+ * The two fields `releasing` and `requested` are used to track the ongoing
+ * state of YARN requests; they do not need to be persisted across stop/start
+ * cycles. They may be relevant across AM restart, but without other data
+ * structures in the AM, not enough to track what the AM was up to before
+ * it was restarted. The strategy will be to ignore unexpected allocation
+ * responses (which may come from pre-restart) requests, while treating
+ * unexpected container release responses as failures.
+ * <p>
+ * The `active` counter is only decremented after a container release response
+ * has been received.
+ * <p>
+ *
*/
public class NodeEntry {
@@ -47,30 +48,45 @@ public class NodeEntry {
/**
* instance explicitly requested on this node: it's OK if an allocation
* comes in that has not been (and when that happens, this count should
- * not drop)
+ * not drop).
*/
private int requested;
+
+ /** number of starting instances */
private int starting;
+
+ /** incrementing counter of instances that failed to start */
private int startFailed;
+
+ /** incrementing counter of instances that failed */
private int failed;
- private int preempted;
+
/**
* Counter of "failed recently" events. These are all failures
* which have happened since it was last reset.
*/
private int failedRecently;
+
+ /** incrementing counter of instances that have been pre-empted. */
+ private int preempted;
+
/**
* Number of live nodes.
*/
private int live;
+
+ /** number of containers being released off this node */
private int releasing;
+
+ /** timestamp of last use */
private long lastUsed;
-
+
/**
- * Is the node available for assignments. This does not track
- * whether or not there are any outstanding requests for this node
- * @return true if there are no role instances here
- * other than some being released.
+ * Is the node available for assignments? That is, it is
+ * not running any instances of this type, nor are there
+ * any requests oustanding for it.
+ * @return true if a new request could be issued without taking
+ * the number of instances > 1.
*/
public synchronized boolean isAvailable() {
return getActive() == 0 && (requested == 0) && starting == 0;
@@ -266,4 +282,23 @@ public class NodeEntry {
sb.append('}');
return sb.toString();
}
+
+ /**
+ * Produced a serialized form which can be served up as JSON
+ * @return a summary of the current role status.
+ */
+ public synchronized NodeEntryInformation serialize() {
+ NodeEntryInformation info = new NodeEntryInformation();
+ info.priority = rolePriority;
+ info.requested = requested;
+ info.releasing = releasing;
+ info.starting = starting;
+ info.startFailed = startFailed;
+ info.failed = failed;
+ info.failedRecently = failedRecently;
+ info.preempted = preempted;
+ info.live = live;
+ info.lastUsed = lastUsed;
+ return info;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index fb80b5f..2afdc42 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -18,6 +18,12 @@
package org.apache.slider.server.appmaster.state;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.slider.api.types.NodeEntryInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.common.tools.SliderUtils;
+
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Comparator;
@@ -33,6 +39,19 @@ public class NodeInstance {
public final String hostname;
+ /**
+ * last state of node. Starts off as {@link NodeState#RUNNING},
+ * on the assumption that it is live.
+ */
+ private NodeState nodeState = NodeState.RUNNING;
+
+ private NodeReport nodeReport = null;
+
+ /**
+ * time of state update
+ */
+ private long nodeStateUpdateTime = 0;
+
private final List<NodeEntry> nodeEntries;
/**
@@ -44,6 +63,20 @@ public class NodeInstance {
nodeEntries = new ArrayList<>(roles);
}
+
+ /**
+ * Update the node status
+ * @param report latest node report
+ * @return true if the node state changed
+ */
+ public boolean updateNode(NodeReport report) {
+ nodeReport = report;
+ NodeState oldState = nodeState;
+ nodeState = report.getNodeState();
+ nodeStateUpdateTime = report.getLastHealthReportTime();
+ return nodeState != oldState;
+ }
+
/**
* Get the entry for a role -if present
* @param role role index
@@ -215,11 +248,35 @@ public class NodeInstance {
}
/**
+ * Produced a serialized form which can be served up as JSON
+ * @return a summary of the current role status.
+ */
+ public synchronized NodeInformation serialize() {
+ NodeInformation info = new NodeInformation();
+ info.hostname = hostname;
+ // null-handling state constructor
+ info.state = "" + nodeState;
+ info.lastUpdated = nodeStateUpdateTime;
+ if (nodeReport != null) {
+ info.httpAddress = nodeReport.getHttpAddress();
+ info.rackName = nodeReport.getRackName();
+ info.labels = SliderUtils.join(nodeReport.getNodeLabels(), ", ", false);
+ info.healthReport = nodeReport.getHealthReport();
+ }
+ info.entries = new ArrayList<>(nodeEntries.size());
+ for (NodeEntry nodeEntry : nodeEntries) {
+ info.entries.add(nodeEntry.serialize());
+ }
+ return info;
+ }
+
+ /**
* A comparator for sorting entries where the node is preferred over another.
* <p>
* The exact algorithm may change
*
- * @return +ve int if left is preferred to right; -ve if right over left, 0 for equal
+ * the comparision is a positive int if left is preferred to right;
+ * negative if right over left, 0 for equal
*/
public static class Preferred implements Comparator<NodeInstance>,
Serializable {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
index aa50baa..b631057 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
@@ -19,6 +19,7 @@
package org.apache.slider.server.appmaster.state;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.api.records.NodeReport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -26,9 +27,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
-import java.util.ListIterator;
import java.util.Map;
/**
@@ -84,27 +83,6 @@ public class NodeMap extends HashMap<String, NodeInstance> {
}
/**
- * purge the history of all nodes that have been inactive since the absolute time
- * @param absoluteTime time
- * @return the number purged
- */
- public int purgeUnusedEntries(long absoluteTime) {
- int purged = 0;
- Iterator<Map.Entry<String, NodeInstance>> iterator =
- entrySet().iterator();
- while (iterator.hasNext()) {
- Map.Entry<String, NodeInstance> entry = iterator.next();
- NodeInstance ni = entry.getValue();
- if (!ni.purgeUnusedEntries(absoluteTime)) {
- iterator.remove();
- purged ++;
- }
- }
- return purged;
- }
-
-
- /**
* reset the failed recently counters
*/
public void resetFailedRecently() {
@@ -114,6 +92,17 @@ public class NodeMap extends HashMap<String, NodeInstance> {
}
}
+
+ /**
+ * Update the node state
+ * @param hostname host name
+ * @param report latest node report
+ * @return the updated node.
+ */
+ public boolean updateNode(String hostname, NodeReport report) {
+ return getOrCreate(hostname).updateNode(report);
+ }
+
/**
* Clone point
* @return
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 78fdcc9..f1e26bb 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.slider.common.tools.SliderUtils;
@@ -66,14 +67,11 @@ public class RoleHistory {
LoggerFactory.getLogger(RoleHistory.class);
private final List<ProviderRole> providerRoles;
private long startTime;
- /**
- * Time when saved
- */
+
+ /** Time when saved */
private long saveTime;
- /**
- * If the history was loaded, the time at which the history was saved
- *
- */
+
+ /** If the history was loaded, the time at which the history was saved */
private long thawedDataTime;
private NodeMap nodemap;
@@ -331,18 +329,6 @@ public class RoleHistory {
}
/**
- * Garbage collect the structure -this will drop
- * all nodes that have been inactive since the (relative) age.
- * This will drop the failure counts of the nodes too, so it will
- * lose information that matters.
- * @param age relative age
- */
- public void gc(long age) {
- long absoluteTime = now() - age;
- purgeUnusedEntries(absoluteTime);
- }
-
- /**
* Mark ourselves as dirty
*/
public void touch() {
@@ -355,15 +341,6 @@ public class RoleHistory {
}
/**
- * purge the history of
- * all nodes that have been inactive since the absolute time
- * @param absoluteTime time
- */
- public synchronized void purgeUnusedEntries(long absoluteTime) {
- nodemap.purgeUnusedEntries(absoluteTime);
- }
-
- /**
* reset the failed recently counters
*/
public synchronized void resetFailedRecently() {
@@ -451,7 +428,7 @@ public class RoleHistory {
if (loadedRoleHistory != null) {
rebuild(loadedRoleHistory);
thawSuccessful = true;
- Path loadPath = loadedRoleHistory.getPath();;
+ Path loadPath = loadedRoleHistory.getPath();
log.debug("loaded history from {}", loadPath);
// delete any old entries
try {
@@ -627,8 +604,8 @@ public class RoleHistory {
}
/**
- * Get the list of active nodes ... walks the node map so
- * is O(nodes)
+ * Get the list of active nodes ... walks the node map so
+ * is {@code O(nodes)}
* @param role role index
* @return a possibly empty list of nodes with an instance of that node
*/
@@ -654,8 +631,7 @@ public class RoleHistory {
* @throws RuntimeException if the container has no hostname
*/
public synchronized NodeInstance getOrCreateNodeInstance(Container container) {
- String hostname = RoleHistoryUtils.hostnameOf(container);
- return nodemap.getOrCreate(hostname);
+ return nodemap.getOrCreate(RoleHistoryUtils.hostnameOf(container));
}
/**
@@ -738,7 +714,7 @@ public class RoleHistory {
}
/**
- * Event: a container start has been submitter
+ * Event: a container start has been submitted
* @param container container being started
* @param instance instance bound to the container
*/
@@ -775,19 +751,25 @@ public class RoleHistory {
* @param updatedNodes list of updated nodes
*/
public synchronized void onNodesUpdated(List<NodeReport> updatedNodes) {
+ log.debug("Updating {} nodes", updatedNodes.size());
for (NodeReport updatedNode : updatedNodes) {
- String hostname = updatedNode.getNodeId() == null
+ String hostname = updatedNode.getNodeId() == null
? null
: updatedNode.getNodeId().getHost();
- if (hostname == null) {
+ NodeState nodeState = updatedNode.getNodeState();
+ if (hostname == null || nodeState == null) {
continue;
}
- if (updatedNode.getNodeState() != null
- && updatedNode.getNodeState().isUnusable()) {
- failedNodes.add(hostname);
- nodemap.remove(hostname);
- } else {
- failedNodes.remove(hostname);
+ // update the node; this also creates an instance if needed
+ boolean updated = nodemap.updateNode(hostname, updatedNode);
+ if (updated) {
+ log.debug("Updated host {} to state {}", hostname, nodeState);
+ if (nodeState.isUnusable()) {
+ log.info("Failed node {}", hostname);
+ failedNodes.add(hostname);
+ } else {
+ failedNodes.remove(hostname);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index e2974fc..98a8311 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -25,6 +25,7 @@ import org.apache.slider.providers.ProviderRole;
import java.io.Serializable;
import java.util.Comparator;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@@ -51,6 +52,15 @@ public final class RoleStatus implements Cloneable {
private final AtomicLong failedRecently = new AtomicLong(0);
private final AtomicLong limitsExceeded = new AtomicLong(0);
+ /** flag set to true if there is an outstanding anti-affine request */
+ private final AtomicBoolean pendingAARequest = new AtomicBoolean(false);
+
+ /**
+ * Number of AA requests queued. These should be reduced first on a
+ * flex down.
+ */
+ private int pendingAntiAffineRequestCount = 0;
+
private String failureMessage = "";
public RoleStatus(ProviderRole providerRole) {
@@ -302,6 +312,7 @@ public final class RoleStatus implements Cloneable {
", actual=" + actual +
", requested=" + requested +
", releasing=" + releasing +
+ ", pendingAntiAffineRequestCount=" + pendingAntiAffineRequestCount +
", failed=" + failed +
", failed recently=" + failedRecently.get() +
", node failed=" + nodeFailed.get() +
@@ -355,6 +366,8 @@ public final class RoleStatus implements Cloneable {
info.failedRecently = failedRecently.intValue();
info.nodeFailed = nodeFailed.intValue();
info.preempted = preempted.intValue();
+ info.pendingAntiAffineRequest = pendingAARequest.get();
+ info.pendingAntiAffineRequestCount = pendingAntiAffineRequestCount;
return info;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
index eef178a..56c7cac 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
@@ -37,7 +37,8 @@ import java.util.List;
import java.util.Map;
/**
- * The methods to offer state access to the providers
+ * The methods to offer state access to the providers and other parts of
+ * the system which want read-only access to the state.
*/
public interface StateAccessForProviders {
@@ -280,5 +281,10 @@ public interface StateAccessForProviders {
*/
List<RoleInstance> lookupRoleContainers(String component);
+ /**
+ * Get the JSON serializable information about a component
+ * @param component component to look up
+ * @return a structure describing the component.
+ */
ComponentInformation getComponentInformation(String component);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
index 179ae8c..0f99d6d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
@@ -81,7 +81,15 @@ public interface WebAppApi {
*/
QueueAccess getQueues();
+ /**
+ * API for AM operations
+ * @return current operations implementation
+ */
AppMasterActionOperations getAMOperations();
+ /**
+ * Local cache of content
+ * @return the cache
+ */
ContentCache getContentCache();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/main/proto/SliderClusterMessages.proto
----------------------------------------------------------------------
diff --git a/slider-core/src/main/proto/SliderClusterMessages.proto b/slider-core/src/main/proto/SliderClusterMessages.proto
index caca87b..8287f36 100644
--- a/slider-core/src/main/proto/SliderClusterMessages.proto
+++ b/slider-core/src/main/proto/SliderClusterMessages.proto
@@ -24,21 +24,28 @@ package org.apache.slider.api;
//import "Security.proto";
+/*
+ To debug compilation problems, bypass the maven build and invoke protoc
+ from the command line
+
+ protoc --java_out=target src/main/proto/SliderClusterMessages.proto
+*/
+
message RoleInstanceState {
- required string name = 1;
- optional string role = 2;
- required uint32 state = 4;
- required uint32 exitCode = 5;
- optional string command = 6;
+ required string name = 1;
+ optional string role = 2;
+ required uint32 state = 4;
+ required uint32 exitCode = 5;
+ optional string command = 6;
optional string diagnostics = 7;
- repeated string output = 8;
+ repeated string output = 8;
repeated string environment = 9;
- required uint32 roleId = 10;
- required bool released = 11;
- required int64 createTime = 12;
- required int64 startTime = 13;
- required string host = 14;
- required string hostURL = 15;
+ required uint32 roleId = 10;
+ required bool released = 11;
+ required int64 createTime = 12;
+ required int64 startTime = 13;
+ required string host = 14;
+ required string hostURL = 15;
optional string appVersion = 16;
}
@@ -65,9 +72,9 @@ message UpgradeContainersRequestProto {
/**
message to include
*/
- required string message = 1;
- repeated string container = 2;
- repeated string component = 3;
+ required string message = 1;
+ repeated string container = 2;
+ repeated string component = 3;
}
/**
@@ -181,9 +188,9 @@ message KillContainerResponseProto {
* AM suicide
*/
message AMSuicideRequestProto {
- required string text = 1;
- required int32 signal = 2;
- required int32 delay = 3;
+ required string text = 1;
+ required int32 signal = 2;
+ required int32 delay = 3;
}
/**
@@ -206,9 +213,9 @@ message GetInstanceDefinitionRequestProto {
* Get the definition back as three separate JSON strings
*/
message GetInstanceDefinitionResponseProto {
- required string internal = 1;
- required string resources = 2;
- required string application = 3;
+ required string internal = 1;
+ required string resources = 2;
+ required string application = 3;
}
@@ -233,42 +240,43 @@ message ApplicationLivenessInformationProto {
* see org.apache.slider.api.types.ComponentInformation
*/
message ComponentInformationProto {
- optional string name = 1;
- optional int32 priority = 2;
- optional int32 desired = 3;
- optional int32 actual = 4;
- optional int32 releasing = 5;
- optional int32 requested = 6;
- optional int32 failed = 7;
- optional int32 started = 8;
- optional int32 startFailed = 9;
- optional int32 completed = 10;
- optional int32 totalRequested = 11;
- optional string failureMessage = 12 ;
- optional int32 placementPolicy = 13;
- repeated string containers = 14;
+ optional string name = 1;
+ optional int32 priority = 2;
+ optional int32 desired = 3;
+ optional int32 actual = 4;
+ optional int32 releasing = 5;
+ optional int32 requested = 6;
+ optional int32 failed = 7;
+ optional int32 started = 8;
+ optional int32 startFailed = 9;
+ optional int32 completed = 10;
+ optional int32 totalRequested = 11;
+ optional string failureMessage =12;
+ optional int32 placementPolicy =13;
+ repeated string containers = 14;
optional int32 failedRecently = 15;
- optional int32 nodeFailed = 16;
- optional int32 preempted = 17;
+ optional int32 nodeFailed = 16;
+ optional int32 preempted = 17;
+ optional int32 pendingAntiAffineRequestCount = 18;
}
/*
* see org.apache.slider.api.types.ContainerInformation
*/
message ContainerInformationProto {
- optional string containerId = 1;
- optional string component = 2;
- optional bool released = 3;
- optional int32 state = 4;
- optional int32 exitCode = 5;
- optional string diagnostics = 6;
- optional int64 createTime = 7;
- optional int64 startTime = 8;
- repeated string output = 9;
- optional string host = 10;
- optional string hostURL = 11;
- optional string placement = 12;
- optional string appVersion = 13;
+ optional string containerId = 1;
+ optional string component = 2;
+ optional bool released = 3;
+ optional int32 state = 4;
+ optional int32 exitCode = 5;
+ optional string diagnostics = 6;
+ optional int64 createTime = 7;
+ optional int64 startTime = 8;
+ repeated string output = 9;
+ optional string host = 10;
+ optional string hostURL = 11;
+ optional string placement = 12;
+ optional string appVersion = 13;
}
@@ -279,10 +287,32 @@ message PingInformationProto {
optional string text = 1;
optional string verb = 2;
optional string body = 3;
- optional int64 time = 4;
+ optional int64 time = 4;
}
+message NodeEntryInformationProto {
+ required int32 priority = 1;
+ required int32 requested = 2;
+ required int32 starting = 3;
+ required int32 startFailed = 4;
+ required int32 failed = 5;
+ required int32 failedRecently= 6;
+ required int32 preempted = 7;
+ required int32 live = 8;
+ required int32 releasing = 9;
+ required int64 lastUsed = 10;
+}
+message NodeInformationProto {
+ required string hostname = 1;
+ required string state = 2;
+ required string httpAddress = 3;
+ required string rackName = 4;
+ required string labels = 5;
+ required string healthReport= 6;
+ required int64 lastUpdated = 7;
+ repeated NodeEntryInformationProto entries = 8;
+}
message GetModelRequestProto {
}
@@ -345,12 +375,12 @@ message WrappedJsonProto {
}
message GetCertificateStoreRequestProto {
- optional string hostname = 1;
+ optional string hostname = 1;
required string requesterId = 2;
- required string password = 3;
- required string type = 4;
+ required string password = 3;
+ required string type = 4;
}
message GetCertificateStoreResponseProto {
required bytes store = 1;
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
index 2cc13c2..6e21a38 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
@@ -19,7 +19,7 @@ package org.apache.slider.server.appmaster.model.mock
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring
-import org.apache.slider.server.appmaster.state.AbstractRecordFactory
+import org.apache.slider.server.appmaster.state.AbstractClusterServices
import org.apache.slider.server.appmaster.state.AppState
/**
@@ -27,7 +27,7 @@ import org.apache.slider.server.appmaster.state.AppState
*/
class MockAppState extends AppState {
- public MockAppState(AbstractRecordFactory recordFactory) {
+ public MockAppState(AbstractClusterServices recordFactory) {
super(recordFactory, new MetricsAndMonitoring());
}
@@ -37,7 +37,7 @@ class MockAppState extends AppState {
* Instance with a mock record factory
*/
public MockAppState() {
- super(new MockRecordFactory(), new MetricsAndMonitoring());
+ super(new MockClusterServices(), new MetricsAndMonitoring());
}
public Map<String, ProviderRole> getRoleMap() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy
new file mode 100644
index 0000000..d27a6bb
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.hadoop.yarn.api.records.Resource
+import org.apache.slider.server.appmaster.state.AbstractClusterServices
+
+class MockClusterServices extends AbstractClusterServices {
+
+ @Override
+ Resource newResource() {
+ return new MockResource()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
deleted file mode 100644
index f7d353f..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock
-
-import org.apache.hadoop.yarn.api.records.Resource
-import org.apache.slider.server.appmaster.state.AbstractRecordFactory
-
-class MockRecordFactory extends AbstractRecordFactory {
-
- @Override
- Resource newResource() {
- return new MockResource()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
index e1964f5..5a7a891 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
@@ -19,15 +19,14 @@ package org.apache.slider.server.appmaster.web.view
import com.google.inject.AbstractModule
import com.google.inject.Guice
import com.google.inject.Injector
-import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet
import org.apache.slider.api.ClusterDescription
import org.apache.slider.providers.ProviderService
import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockProviderService
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
-import org.apache.slider.server.appmaster.state.AbstractRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockClusterServices
+import org.apache.slider.server.appmaster.state.AbstractClusterServices
import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.ProviderAppState
import org.apache.slider.server.appmaster.web.WebAppApi
@@ -43,7 +42,7 @@ public class TestClusterSpecificationBlock {
@Before
public void setup() {
- AppState appState = new MyAppState(new MockRecordFactory());
+ AppState appState = new MyAppState(new MockClusterServices());
ProviderAppState providerAppState = new ProviderAppState(
"undefined",
appState)
@@ -79,7 +78,7 @@ public class TestClusterSpecificationBlock {
}
private static class MyAppState extends MockAppState {
- public MyAppState(AbstractRecordFactory recordFactory) {
+ public MyAppState(AbstractClusterServices recordFactory) {
super(recordFactory);
this.clusterStatus = new MockClusterDescription();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
index d37fcea..4c43168 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -37,7 +37,7 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.model.mock.MockFactory;
import org.apache.slider.server.appmaster.model.mock.MockProviderService;
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory;
+import org.apache.slider.server.appmaster.model.mock.MockClusterServices;
import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
@@ -126,7 +126,7 @@ public class TestAMAgentWebServices {
historyPath =
new org.apache.hadoop.fs.Path(historyWorkDir.toURI());
fs.delete(historyPath, true);
- appState = new AppState(new MockRecordFactory(), new MetricsAndMonitoring());
+ appState = new AppState(new MockClusterServices(), new MetricsAndMonitoring());
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES);
appState.buildInstance(
factory.newInstanceDefinition(0, 0, 0),
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/de040cbb/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
index 7cb9395..df7e002 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
@@ -42,7 +42,7 @@ import org.apache.slider.core.persist.JsonSerDeser;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.model.mock.MockFactory;
import org.apache.slider.server.appmaster.model.mock.MockProviderService;
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory;
+import org.apache.slider.server.appmaster.model.mock.MockClusterServices;
import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
@@ -167,7 +167,7 @@ public class TestAMManagementWebServices extends JerseyTest {
historyPath =
new org.apache.hadoop.fs.Path(historyWorkDir.toURI());
fs.delete(historyPath, true);
- appState = new AppState(new MockRecordFactory(), new MetricsAndMonitoring());
+ appState = new AppState(new MockClusterServices(), new MetricsAndMonitoring());
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES);
appState.buildInstance(
factory.newInstanceDefinition(0, 0, 0),