You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2017/11/09 21:46:42 UTC
[3/4] storm git commit: STORM-2725: Generic Resource Aware Scheduling
STORM-2725: Generic Resource Aware Scheduling
STORM-2725: Generic Resource Scheduling - initial config changes and TopologyBuilder API
STORM-2725: Generic Resource Aware Scheduling(GRAS) - metadata, scheduling strategies and configuration enhancements
STORM-2725: Generic Resource Aware Scheduling(GRAS) - metadata, scheduling strategies and configuration enhancements
STORM-2725: Checkstyle stuff
STORM-2725: GRAS tests
STORM-2725: Feedback
STORM-2725: Merging with revans2 scheduler cleanups
STORM-2725: Leftover feedback
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8a9141c2
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8a9141c2
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8a9141c2
Branch: refs/heads/master
Commit: 8a9141c24d2732865061765a43cd8e1871709095
Parents: 566db74
Author: Govind Menon <go...@gmail.com>
Authored: Thu Sep 7 13:50:05 2017 -0500
Committer: Govind Menon <go...@gmail.com>
Committed: Thu Nov 9 13:56:08 2017 -0600
----------------------------------------------------------------------
.../org/apache/storm/loadgen/CaptureLoad.java | 66 +-
.../src/jvm/org/apache/storm/Config.java | 12 +
.../src/jvm/org/apache/storm/Constants.java | 24 +-
.../coordination/BatchSubtopologyBuilder.java | 62 +-
.../storm/drpc/LinearDRPCTopologyBuilder.java | 87 ++-
.../apache/storm/generated/WorkerResources.java | 324 ++++++++-
.../storm/scheduler/SupervisorDetails.java | 37 +-
.../topology/BaseConfigurationDeclarer.java | 14 +-
.../ComponentConfigurationDeclarer.java | 30 +-
.../apache/storm/topology/TopologyBuilder.java | 25 +-
.../TransactionalTopologyBuilder.java | 58 +-
.../topology/TridentTopologyBuilder.java | 48 ++
.../jvm/org/apache/storm/utils/ConfigUtils.java | 18 +-
.../org/apache/storm/utils/ObjectReader.java | 2 +-
storm-client/src/storm.thrift | 2 +
.../java/org/apache/storm/LocalCluster.java | 31 +-
.../org/apache/storm/daemon/nimbus/Nimbus.java | 9 +-
.../supervisor/timer/SupervisorHeartbeat.java | 18 +-
.../storm/logging/ThriftAccessLogger.java | 2 +
.../org/apache/storm/scheduler/Cluster.java | 178 +++--
.../storm/scheduler/ISchedulingState.java | 11 +-
.../apache/storm/scheduler/TopologyDetails.java | 87 ++-
.../storm/scheduler/resource/RAS_Node.java | 70 +-
.../storm/scheduler/resource/ResourceUtils.java | 179 +++--
.../scheduling/BaseResourceAwareStrategy.java | 610 +++++++++++++++++
.../DefaultResourceAwareStrategy.java | 677 ++-----------------
.../GenericResourceAwareStrategy.java | 193 ++++++
.../apache/storm/utils/BufferInputStream.java | 7 +-
.../apache/storm/utils/ServerConfigUtils.java | 14 +-
.../org/apache/storm/utils/ServerUtils.java | 65 +-
.../resource/TestResourceAwareScheduler.java | 15 +-
.../TestUtilsForResourceAwareScheduler.java | 36 +-
.../TestDefaultResourceAwareStrategy.java | 8 +-
.../TestGenericResourceAwareStrategy.java | 209 ++++++
34 files changed, 2282 insertions(+), 946 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/CaptureLoad.java
----------------------------------------------------------------------
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/CaptureLoad.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/CaptureLoad.java
index 11d64a1..5c1ef4e 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/CaptureLoad.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/CaptureLoad.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
-import java.util.stream.Stream;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
@@ -108,7 +107,7 @@ public class CaptureLoad {
//Lets use the number of actually scheduled workers as a way to bridge RAS and non-RAS
int numWorkers = tpinfo.get_num_workers();
if (savedTopoConf.containsKey(Config.TOPOLOGY_WORKERS)) {
- numWorkers = Math.max(numWorkers, ((Number)savedTopoConf.get(Config.TOPOLOGY_WORKERS)).intValue());
+ numWorkers = Math.max(numWorkers, ((Number) savedTopoConf.get(Config.TOPOLOGY_WORKERS)).intValue());
}
savedTopoConf.put(Config.TOPOLOGY_WORKERS, numWorkers);
@@ -372,26 +371,24 @@ public class CaptureLoad {
//ResourceUtils.java is not a available on the classpath to let us parse out the resources we want.
// So we have copied and pasted some of the needed methods here. (with a few changes to logging)
- static Map<String, Map<String, Double>> getBoltsResources(StormTopology topology,
- Map<String, Object> topologyConf) {
+ static Map<String, Map<String, Double>> getBoltsResources(StormTopology topology, Map<String, Object> topologyConf) {
Map<String, Map<String, Double>> boltResources = new HashMap<>();
if (topology.get_bolts() != null) {
for (Map.Entry<String, Bolt> bolt : topology.get_bolts().entrySet()) {
Map<String, Double> topologyResources = parseResources(bolt.getValue().get_common().get_json_conf());
- checkIntialization(topologyResources, bolt.getValue().toString(), topologyConf);
+ checkInitialization(topologyResources, bolt.getValue().toString(), topologyConf);
boltResources.put(bolt.getKey(), topologyResources);
}
}
return boltResources;
}
- static Map<String, Map<String, Double>> getSpoutsResources(StormTopology topology,
- Map<String, Object> topologyConf) {
+ static Map<String, Map<String, Double>> getSpoutsResources(StormTopology topology, Map<String, Object> topologyConf) {
Map<String, Map<String, Double>> spoutResources = new HashMap<>();
if (topology.get_spouts() != null) {
for (Map.Entry<String, SpoutSpec> spout : topology.get_spouts().entrySet()) {
Map<String, Double> topologyResources = parseResources(spout.getValue().get_common().get_json_conf());
- checkIntialization(topologyResources, spout.getValue().toString(), topologyConf);
+ checkInitialization(topologyResources, spout.getValue().toString(), topologyConf);
spoutResources.put(spout.getKey(), topologyResources);
}
}
@@ -430,37 +427,40 @@ public class CaptureLoad {
return topologyResources;
}
- static void checkIntialization(Map<String, Double> topologyResources, String com,
- Map<String, Object> topologyConf) {
- checkInitMem(topologyResources, com, topologyConf);
- checkInitCpu(topologyResources, com, topologyConf);
- }
+ /**
+ * Checks if the topology's resource requirements are initialized.
+ * Will modify topologyResources by adding the appropriate defaults
+ * @param topologyResources map of resouces requirements
+ * @param componentId component for which initialization is being conducted
+ * @param topologyConf topology configuration
+ * @throws Exception on any error
+ */
+ public static void checkInitialization(Map<String, Double> topologyResources, String componentId, Map topologyConf) {
+ StringBuilder msgBuilder = new StringBuilder();
- static void checkInitMem(Map<String, Double> topologyResources, String com,
- Map<String, Object> topologyConf) {
- if (!topologyResources.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)) {
- Double onHeap = ObjectReader.getDouble(
- topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB), null);
- if (onHeap != null) {
- topologyResources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, onHeap);
- }
+ for (String resourceName : topologyResources.keySet()) {
+ msgBuilder.append(checkInitResource(topologyResources, topologyConf, resourceName));
}
- if (!topologyResources.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB)) {
- Double offHeap = ObjectReader.getDouble(
- topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB), null);
- if (offHeap != null) {
- topologyResources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, offHeap);
- }
+
+ if (msgBuilder.length() > 0) {
+ String resourceDefaults = msgBuilder.toString();
+ LOG.debug(
+ "Unable to extract resource requirement for Component {} \n Resources : {}",
+ componentId, resourceDefaults);
}
}
- static void checkInitCpu(Map<String, Double> topologyResources, String com,
- Map<String, Object> topologyConf) {
- if (!topologyResources.containsKey(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT)) {
- Double cpu = ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT), null);
- if (cpu != null) {
- topologyResources.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, cpu);
+ private static String checkInitResource(Map<String, Double> topologyResources, Map topologyConf, String resourceName) {
+ StringBuilder msgBuilder = new StringBuilder();
+ if (topologyResources.containsKey(resourceName)) {
+ Double resourceValue = (Double) topologyConf.getOrDefault(resourceName, null);
+ if (resourceValue != null) {
+ topologyResources.put(resourceName, resourceValue);
+ msgBuilder.append(resourceName.substring(resourceName.lastIndexOf(".")) + " has been set to " + resourceValue);
}
}
+
+ return msgBuilder.toString();
}
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-client/src/jvm/org/apache/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/Config.java b/storm-client/src/jvm/org/apache/storm/Config.java
index 42a4b44..cc47e14 100644
--- a/storm-client/src/jvm/org/apache/storm/Config.java
+++ b/storm-client/src/jvm/org/apache/storm/Config.java
@@ -241,6 +241,12 @@ public class Config extends HashMap<String, Object> {
public static final String TOPOLOGY_TASKS = "topology.tasks";
/**
+ * A map of resources used by each component e.g {"cpu.pcore.percent" : 200.0. "onheap.memory.mb": 256.0, "gpu.count" : 2 }
+ */
+ @isMapEntryType(keyType = String.class, valueType = Number.class)
+ public static final String TOPOLOGY_COMPONENT_RESOURCES_MAP = "topology.component.resources.map";
+
+ /**
* The maximum amount of memory an instance of a spout/bolt will take on heap. This enables the scheduler
* to allocate slots on machines with enough available memory. A default value will be set for this config if user does not override
*/
@@ -1288,6 +1294,12 @@ public class Config extends HashMap<String, Object> {
public static final String SUPERVISOR_CPU_CAPACITY = "supervisor.cpu.capacity";
/**
+ * A map of resources the Supervisor has e.g {"cpu.pcore.percent" : 200.0. "onheap.memory.mb": 256.0, "gpu.count" : 2.0 }
+ */
+ @isMapEntryType(keyType = String.class, valueType = Number.class)
+ public static final String SUPERVISOR_RESOURCES_MAP = "supervisor.resources.map";
+
+ /**
* Whether or not to use ZeroMQ for messaging in local mode. If this is set
* to false, then Storm will use a pure-Java messaging system. The purpose
* of this flag is to make it easy to run Storm in local mode by eliminating
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-client/src/jvm/org/apache/storm/Constants.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/Constants.java b/storm-client/src/jvm/org/apache/storm/Constants.java
index 7375c70..c8d3b09 100644
--- a/storm-client/src/jvm/org/apache/storm/Constants.java
+++ b/storm-client/src/jvm/org/apache/storm/Constants.java
@@ -15,12 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.storm;
-import org.apache.storm.coordination.CoordinatedBolt;
+package org.apache.storm;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import org.apache.storm.coordination.CoordinatedBolt;
public class Constants {
@@ -56,5 +59,22 @@ public class Constants {
public static final String STORM_ACTIVE_ATOM = "storm-active-atom";
public static final String COMPONENT_TO_DEBUG_ATOM = "storm-component->debug-atom";
public static final Object LOAD_MAPPING = "load-mapping";
+
+ public static final String COMMON_CPU_RESOURCE_NAME = "cpu.pcore.percent";
+ public static final String COMMON_ONHEAP_MEMORY_RESOURCE_NAME = "onheap.memory.mb";
+ public static final String COMMON_OFFHEAP_MEMORY_RESOURCE_NAME = "offheap.memory.mb";
+ public static final String COMMON_TOTAL_MEMORY_RESOURCE_NAME = "memory.mb";
+
+ public static final Map<String, String> resourceNameMapping;
+
+ static {
+ Map<String, String> tmp = new HashMap<>();
+ tmp.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, COMMON_CPU_RESOURCE_NAME);
+ tmp.put(Config.SUPERVISOR_CPU_CAPACITY, COMMON_CPU_RESOURCE_NAME);
+ tmp.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, COMMON_ONHEAP_MEMORY_RESOURCE_NAME);
+ tmp.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, COMMON_OFFHEAP_MEMORY_RESOURCE_NAME);
+ tmp.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, COMMON_TOTAL_MEMORY_RESOURCE_NAME);
+ resourceNameMapping = Collections.unmodifiableMap(tmp);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-client/src/jvm/org/apache/storm/coordination/BatchSubtopologyBuilder.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/coordination/BatchSubtopologyBuilder.java b/storm-client/src/jvm/org/apache/storm/coordination/BatchSubtopologyBuilder.java
index 142e5f9..9318aa8 100644
--- a/storm-client/src/jvm/org/apache/storm/coordination/BatchSubtopologyBuilder.java
+++ b/storm-client/src/jvm/org/apache/storm/coordination/BatchSubtopologyBuilder.java
@@ -15,8 +15,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.coordination;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.storm.Config;
import org.apache.storm.Constants;
import org.apache.storm.coordination.CoordinatedBolt.SourceArgs;
import org.apache.storm.generated.GlobalStreamId;
@@ -32,12 +41,7 @@ import org.apache.storm.topology.IRichBolt;
import org.apache.storm.topology.InputDeclarer;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+
public class BatchSubtopologyBuilder {
Map<String, Component> _bolts = new HashMap<String, Component>();
@@ -76,7 +80,9 @@ public class BatchSubtopologyBuilder {
private BoltDeclarer setBolt(String id, IRichBolt bolt, Number parallelism) {
Integer p = null;
- if(parallelism!=null) p = parallelism.intValue();
+ if (parallelism != null) {
+ p = parallelism.intValue();
+ }
Component component = new Component(bolt, p);
_bolts.put(id, component);
return new BoltDeclarerImpl(component);
@@ -84,18 +90,18 @@ public class BatchSubtopologyBuilder {
public void extendTopology(TopologyBuilder builder) {
BoltDeclarer declarer = builder.setBolt(_masterId, new CoordinatedBolt(_masterBolt.bolt), _masterBolt.parallelism);
- for(InputDeclaration decl: _masterBolt.declarations) {
+ for (InputDeclaration decl: _masterBolt.declarations) {
decl.declare(declarer);
}
- for(Map<String, Object> conf: _masterBolt.componentConfs) {
+ for (Map<String, Object> conf: _masterBolt.componentConfs) {
declarer.addConfigurations(conf);
}
- for(String id: _bolts.keySet()) {
+ for (String id: _bolts.keySet()) {
Component component = _bolts.get(id);
Map<String, SourceArgs> coordinatedArgs = new HashMap<String, SourceArgs>();
- for(String c: componentBoltSubscriptions(component)) {
+ for (String c: componentBoltSubscriptions(component)) {
SourceArgs source;
- if(c.equals(_masterId)) {
+ if (c.equals(_masterId)) {
source = SourceArgs.single();
} else {
source = SourceArgs.all();
@@ -112,13 +118,13 @@ public class BatchSubtopologyBuilder {
for (SharedMemory request: component.sharedMemory) {
input.addSharedMemory(request);
}
- for(Map<String, Object> conf: component.componentConfs) {
+ for (Map<String, Object> conf: component.componentConfs) {
input.addConfigurations(conf);
}
- for(String c: componentBoltSubscriptions(component)) {
+ for (String c: componentBoltSubscriptions(component)) {
input.directGrouping(c, Constants.COORDINATED_STREAM_ID);
}
- for(InputDeclaration d: component.declarations) {
+ for (InputDeclaration d: component.declarations) {
d.declare(input);
}
}
@@ -126,7 +132,7 @@ public class BatchSubtopologyBuilder {
private Set<String> componentBoltSubscriptions(Component component) {
Set<String> ret = new HashSet<String>();
- for(InputDeclaration d: component.declarations) {
+ for (InputDeclaration d: component.declarations) {
ret.add(d.getComponent());
}
return ret;
@@ -450,9 +456,33 @@ public class BatchSubtopologyBuilder {
}
@Override
+ public Map getRASConfiguration() {
+ for (Map<String, Object> conf : _component.componentConfs) {
+ if (conf.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP)) {
+ return conf;
+ }
+ }
+ Map<String, Object> newConf = new HashMap<>();
+ newConf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, new HashMap());
+ _component.componentConfs.add(newConf);
+ return newConf;
+ }
+
+ @Override
public BoltDeclarer addSharedMemory(SharedMemory request) {
_component.sharedMemory.add(request);
return this;
}
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public BoltDeclarer addResource(String resourceName, Number resourceValue) {
+ Map<String, Double> resourcesMap = (Map<String, Double>) getRASConfiguration().get(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP);
+
+ resourcesMap.put(resourceName, resourceValue.doubleValue());
+
+ getRASConfiguration().put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, resourcesMap);
+ return this;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-client/src/jvm/org/apache/storm/drpc/LinearDRPCTopologyBuilder.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/drpc/LinearDRPCTopologyBuilder.java b/storm-client/src/jvm/org/apache/storm/drpc/LinearDRPCTopologyBuilder.java
index 8fe85f3..928a5fa 100644
--- a/storm-client/src/jvm/org/apache/storm/drpc/LinearDRPCTopologyBuilder.java
+++ b/storm-client/src/jvm/org/apache/storm/drpc/LinearDRPCTopologyBuilder.java
@@ -15,8 +15,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.drpc;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.storm.Config;
import org.apache.storm.Constants;
import org.apache.storm.ILocalDRPC;
import org.apache.storm.coordination.BatchBoltExecutor;
@@ -39,12 +48,6 @@ import org.apache.storm.topology.InputDeclarer;
import org.apache.storm.topology.OutputFieldsGetter;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
// Trident subsumes the functionality provided by this class, so it's deprecated
@@ -68,7 +71,9 @@ public class LinearDRPCTopologyBuilder {
@Deprecated
public LinearDRPCInputDeclarer addBolt(IRichBolt bolt, Number parallelism) {
- if(parallelism==null) parallelism = 1;
+ if (parallelism == null) {
+ parallelism = 1;
+ }
Component component = new Component(bolt, parallelism.intValue());
_components.add(component);
return new InputDeclarerImpl(component);
@@ -104,18 +109,18 @@ public class LinearDRPCTopologyBuilder {
builder.setSpout(SPOUT_ID, spout);
builder.setBolt(PREPARE_ID, new PrepareRequest())
.noneGrouping(SPOUT_ID);
- int i=0;
- for(; i<_components.size();i++) {
+ int i = 0;
+ for (; i < _components.size();i++) {
Component component = _components.get(i);
Map<String, SourceArgs> source = new HashMap<String, SourceArgs>();
- if (i==1) {
- source.put(boltId(i-1), SourceArgs.single());
- } else if (i>=2) {
- source.put(boltId(i-1), SourceArgs.all());
+ if (i == 1) {
+ source.put(boltId(i - 1), SourceArgs.single());
+ } else if (i >= 2) {
+ source.put(boltId(i - 1), SourceArgs.all());
}
IdStreamSpec idSpec = null;
- if(i==_components.size()-1 && component.bolt instanceof FinishedCallback) {
+ if (i == _components.size() - 1 && component.bolt instanceof FinishedCallback) {
idSpec = IdStreamSpec.makeDetectSpec(PREPARE_ID, PrepareRequest.ID_STREAM);
}
BoltDeclarer declarer = builder.setBolt(
@@ -127,50 +132,52 @@ public class LinearDRPCTopologyBuilder {
declarer.addSharedMemory(request);
}
- for(Map<String, Object> conf: component.componentConfs) {
+ for (Map<String, Object> conf: component.componentConfs) {
declarer.addConfigurations(conf);
}
- if(idSpec!=null) {
+ if (idSpec != null) {
declarer.fieldsGrouping(idSpec.getGlobalStreamId().get_componentId(), PrepareRequest.ID_STREAM, new Fields("request"));
}
- if(i==0 && component.declarations.isEmpty()) {
+ if (i == 0 && component.declarations.isEmpty()) {
declarer.noneGrouping(PREPARE_ID, PrepareRequest.ARGS_STREAM);
} else {
String prevId;
- if(i==0) {
+ if (i == 0) {
prevId = PREPARE_ID;
} else {
- prevId = boltId(i-1);
+ prevId = boltId(i - 1);
}
- for(InputDeclaration declaration: component.declarations) {
+ for (InputDeclaration declaration: component.declarations) {
declaration.declare(prevId, declarer);
}
}
- if(i>0) {
- declarer.directGrouping(boltId(i-1), Constants.COORDINATED_STREAM_ID);
+ if (i > 0) {
+ declarer.directGrouping(boltId(i - 1), Constants.COORDINATED_STREAM_ID);
}
}
- IRichBolt lastBolt = _components.get(_components.size()-1).bolt;
+ IRichBolt lastBolt = _components.get(_components.size() - 1).bolt;
OutputFieldsGetter getter = new OutputFieldsGetter();
lastBolt.declareOutputFields(getter);
Map<String, StreamInfo> streams = getter.getFieldsDeclaration();
- if(streams.size()!=1) {
+ if (streams.size() != 1) {
throw new RuntimeException("Must declare exactly one stream from last bolt in LinearDRPCTopology");
}
String outputStream = streams.keySet().iterator().next();
List<String> fields = streams.get(outputStream).get_output_fields();
- if(fields.size()!=2) {
- throw new RuntimeException("Output stream of last component in LinearDRPCTopology must contain exactly two fields. The first should be the request id, and the second should be the result.");
+ if (fields.size() != 2) {
+ throw new RuntimeException(
+ "Output stream of last component in LinearDRPCTopology must contain exactly two fields. "
+ + "The first should be the request id, and the second should be the result.");
}
builder.setBolt(boltId(i), new JoinResult(PREPARE_ID))
- .fieldsGrouping(boltId(i-1), outputStream, new Fields(fields.get(0)))
+ .fieldsGrouping(boltId(i - 1), outputStream, new Fields(fields.get(0)))
.fieldsGrouping(PREPARE_ID, PrepareRequest.RETURN_STREAM, new Fields("request"));
i++;
builder.setBolt(boltId(i), new ReturnResults())
- .noneGrouping(boltId(i-1));
+ .noneGrouping(boltId(i - 1));
return builder.createTopology();
}
@@ -398,6 +405,30 @@ public class LinearDRPCTopologyBuilder {
return this;
}
+ @SuppressWarnings("unchecked")
+ @Override
+ public LinearDRPCInputDeclarer addResource(String resourceName, Number resourceValue) {
+ Map<String, Double> resourcesMap = (Map<String, Double>) getRASConfiguration().get(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP);
+
+ resourcesMap.put(resourceName, resourceValue.doubleValue());
+
+ getRASConfiguration().put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, resourcesMap);
+ return this;
+ }
+
+ @Override
+ public Map getRASConfiguration() {
+ for (Map<String, Object> conf : _component.componentConfs) {
+ if (conf.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP)) {
+ return conf;
+ }
+ }
+ Map<String, Object> newConf = new HashMap<>();
+ newConf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, new HashMap());
+ _component.componentConfs.add(newConf);
+ return newConf;
+ }
+
@Override
public LinearDRPCInputDeclarer addSharedMemory(SharedMemory request) {
_component.sharedMemory.add(request);
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-client/src/jvm/org/apache/storm/generated/WorkerResources.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/generated/WorkerResources.java b/storm-client/src/jvm/org/apache/storm/generated/WorkerResources.java
index 16c0a21..56e2348 100644
--- a/storm-client/src/jvm/org/apache/storm/generated/WorkerResources.java
+++ b/storm-client/src/jvm/org/apache/storm/generated/WorkerResources.java
@@ -60,6 +60,8 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
private static final org.apache.thrift.protocol.TField CPU_FIELD_DESC = new org.apache.thrift.protocol.TField("cpu", org.apache.thrift.protocol.TType.DOUBLE, (short)3);
private static final org.apache.thrift.protocol.TField SHARED_MEM_ON_HEAP_FIELD_DESC = new org.apache.thrift.protocol.TField("shared_mem_on_heap", org.apache.thrift.protocol.TType.DOUBLE, (short)4);
private static final org.apache.thrift.protocol.TField SHARED_MEM_OFF_HEAP_FIELD_DESC = new org.apache.thrift.protocol.TField("shared_mem_off_heap", org.apache.thrift.protocol.TType.DOUBLE, (short)5);
+ private static final org.apache.thrift.protocol.TField RESOURCES_FIELD_DESC = new org.apache.thrift.protocol.TField("resources", org.apache.thrift.protocol.TType.MAP, (short)6);
+ private static final org.apache.thrift.protocol.TField SHARED_RESOURCES_FIELD_DESC = new org.apache.thrift.protocol.TField("shared_resources", org.apache.thrift.protocol.TType.MAP, (short)7);
private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
static {
@@ -72,6 +74,8 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
private double cpu; // optional
private double shared_mem_on_heap; // optional
private double shared_mem_off_heap; // optional
+ private Map<String,Double> resources; // optional
+ private Map<String,Double> shared_resources; // optional
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -79,7 +83,9 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
MEM_OFF_HEAP((short)2, "mem_off_heap"),
CPU((short)3, "cpu"),
SHARED_MEM_ON_HEAP((short)4, "shared_mem_on_heap"),
- SHARED_MEM_OFF_HEAP((short)5, "shared_mem_off_heap");
+ SHARED_MEM_OFF_HEAP((short)5, "shared_mem_off_heap"),
+ RESOURCES((short)6, "resources"),
+ SHARED_RESOURCES((short)7, "shared_resources");
private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
@@ -104,6 +110,10 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
return SHARED_MEM_ON_HEAP;
case 5: // SHARED_MEM_OFF_HEAP
return SHARED_MEM_OFF_HEAP;
+ case 6: // RESOURCES
+ return RESOURCES;
+ case 7: // SHARED_RESOURCES
+ return SHARED_RESOURCES;
default:
return null;
}
@@ -150,7 +160,7 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
private static final int __SHARED_MEM_ON_HEAP_ISSET_ID = 3;
private static final int __SHARED_MEM_OFF_HEAP_ISSET_ID = 4;
private byte __isset_bitfield = 0;
- private static final _Fields optionals[] = {_Fields.MEM_ON_HEAP,_Fields.MEM_OFF_HEAP,_Fields.CPU,_Fields.SHARED_MEM_ON_HEAP,_Fields.SHARED_MEM_OFF_HEAP};
+ private static final _Fields optionals[] = {_Fields.MEM_ON_HEAP,_Fields.MEM_OFF_HEAP,_Fields.CPU,_Fields.SHARED_MEM_ON_HEAP,_Fields.SHARED_MEM_OFF_HEAP,_Fields.RESOURCES,_Fields.SHARED_RESOURCES};
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -164,6 +174,14 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
tmpMap.put(_Fields.SHARED_MEM_OFF_HEAP, new org.apache.thrift.meta_data.FieldMetaData("shared_mem_off_heap", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
+ tmpMap.put(_Fields.RESOURCES, new org.apache.thrift.meta_data.FieldMetaData("resources", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE))));
+ tmpMap.put(_Fields.SHARED_RESOURCES, new org.apache.thrift.meta_data.FieldMetaData("shared_resources", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE))));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(WorkerResources.class, metaDataMap);
}
@@ -181,6 +199,14 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
this.cpu = other.cpu;
this.shared_mem_on_heap = other.shared_mem_on_heap;
this.shared_mem_off_heap = other.shared_mem_off_heap;
+ if (other.is_set_resources()) {
+ Map<String,Double> __this__resources = new HashMap<String,Double>(other.resources);
+ this.resources = __this__resources;
+ }
+ if (other.is_set_shared_resources()) {
+ Map<String,Double> __this__shared_resources = new HashMap<String,Double>(other.shared_resources);
+ this.shared_resources = __this__shared_resources;
+ }
}
public WorkerResources deepCopy() {
@@ -199,6 +225,8 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
this.shared_mem_on_heap = 0.0;
set_shared_mem_off_heap_isSet(false);
this.shared_mem_off_heap = 0.0;
+ this.resources = null;
+ this.shared_resources = null;
}
public double get_mem_on_heap() {
@@ -311,6 +339,74 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
__isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __SHARED_MEM_OFF_HEAP_ISSET_ID, value);
}
+ public int get_resources_size() {
+ return (this.resources == null) ? 0 : this.resources.size();
+ }
+
+ public void put_to_resources(String key, double val) {
+ if (this.resources == null) {
+ this.resources = new HashMap<String,Double>();
+ }
+ this.resources.put(key, val);
+ }
+
+ public Map<String,Double> get_resources() {
+ return this.resources;
+ }
+
+ public void set_resources(Map<String,Double> resources) {
+ this.resources = resources;
+ }
+
+ public void unset_resources() {
+ this.resources = null;
+ }
+
+ /** Returns true if field resources is set (has been assigned a value) and false otherwise */
+ public boolean is_set_resources() {
+ return this.resources != null;
+ }
+
+ public void set_resources_isSet(boolean value) {
+ if (!value) {
+ this.resources = null;
+ }
+ }
+
+ public int get_shared_resources_size() {
+ return (this.shared_resources == null) ? 0 : this.shared_resources.size();
+ }
+
+ public void put_to_shared_resources(String key, double val) {
+ if (this.shared_resources == null) {
+ this.shared_resources = new HashMap<String,Double>();
+ }
+ this.shared_resources.put(key, val);
+ }
+
+ public Map<String,Double> get_shared_resources() {
+ return this.shared_resources;
+ }
+
+ public void set_shared_resources(Map<String,Double> shared_resources) {
+ this.shared_resources = shared_resources;
+ }
+
+ public void unset_shared_resources() {
+ this.shared_resources = null;
+ }
+
+ /** Returns true if field shared_resources is set (has been assigned a value) and false otherwise */
+ public boolean is_set_shared_resources() {
+ return this.shared_resources != null;
+ }
+
+ public void set_shared_resources_isSet(boolean value) {
+ if (!value) {
+ this.shared_resources = null;
+ }
+ }
+
public void setFieldValue(_Fields field, Object value) {
switch (field) {
case MEM_ON_HEAP:
@@ -353,6 +449,22 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
}
break;
+ case RESOURCES:
+ if (value == null) {
+ unset_resources();
+ } else {
+ set_resources((Map<String,Double>)value);
+ }
+ break;
+
+ case SHARED_RESOURCES:
+ if (value == null) {
+ unset_shared_resources();
+ } else {
+ set_shared_resources((Map<String,Double>)value);
+ }
+ break;
+
}
}
@@ -373,6 +485,12 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
case SHARED_MEM_OFF_HEAP:
return get_shared_mem_off_heap();
+ case RESOURCES:
+ return get_resources();
+
+ case SHARED_RESOURCES:
+ return get_shared_resources();
+
}
throw new IllegalStateException();
}
@@ -394,6 +512,10 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
return is_set_shared_mem_on_heap();
case SHARED_MEM_OFF_HEAP:
return is_set_shared_mem_off_heap();
+ case RESOURCES:
+ return is_set_resources();
+ case SHARED_RESOURCES:
+ return is_set_shared_resources();
}
throw new IllegalStateException();
}
@@ -456,6 +578,24 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
return false;
}
+ boolean this_present_resources = true && this.is_set_resources();
+ boolean that_present_resources = true && that.is_set_resources();
+ if (this_present_resources || that_present_resources) {
+ if (!(this_present_resources && that_present_resources))
+ return false;
+ if (!this.resources.equals(that.resources))
+ return false;
+ }
+
+ boolean this_present_shared_resources = true && this.is_set_shared_resources();
+ boolean that_present_shared_resources = true && that.is_set_shared_resources();
+ if (this_present_shared_resources || that_present_shared_resources) {
+ if (!(this_present_shared_resources && that_present_shared_resources))
+ return false;
+ if (!this.shared_resources.equals(that.shared_resources))
+ return false;
+ }
+
return true;
}
@@ -488,6 +628,16 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
if (present_shared_mem_off_heap)
list.add(shared_mem_off_heap);
+ boolean present_resources = true && (is_set_resources());
+ list.add(present_resources);
+ if (present_resources)
+ list.add(resources);
+
+ boolean present_shared_resources = true && (is_set_shared_resources());
+ list.add(present_shared_resources);
+ if (present_shared_resources)
+ list.add(shared_resources);
+
return list.hashCode();
}
@@ -549,6 +699,26 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
return lastComparison;
}
}
+ lastComparison = Boolean.valueOf(is_set_resources()).compareTo(other.is_set_resources());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (is_set_resources()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.resources, other.resources);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(is_set_shared_resources()).compareTo(other.is_set_shared_resources());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (is_set_shared_resources()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.shared_resources, other.shared_resources);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
return 0;
}
@@ -598,6 +768,26 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
sb.append(this.shared_mem_off_heap);
first = false;
}
+ if (is_set_resources()) {
+ if (!first) sb.append(", ");
+ sb.append("resources:");
+ if (this.resources == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.resources);
+ }
+ first = false;
+ }
+ if (is_set_shared_resources()) {
+ if (!first) sb.append(", ");
+ sb.append("shared_resources:");
+ if (this.shared_resources == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.shared_resources);
+ }
+ first = false;
+ }
sb.append(")");
return sb.toString();
}
@@ -683,6 +873,46 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
+ case 6: // RESOURCES
+ if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
+ {
+ org.apache.thrift.protocol.TMap _map646 = iprot.readMapBegin();
+ struct.resources = new HashMap<String,Double>(2*_map646.size);
+ String _key647;
+ double _val648;
+ for (int _i649 = 0; _i649 < _map646.size; ++_i649)
+ {
+ _key647 = iprot.readString();
+ _val648 = iprot.readDouble();
+ struct.resources.put(_key647, _val648);
+ }
+ iprot.readMapEnd();
+ }
+ struct.set_resources_isSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 7: // SHARED_RESOURCES
+ if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
+ {
+ org.apache.thrift.protocol.TMap _map650 = iprot.readMapBegin();
+ struct.shared_resources = new HashMap<String,Double>(2*_map650.size);
+ String _key651;
+ double _val652;
+ for (int _i653 = 0; _i653 < _map650.size; ++_i653)
+ {
+ _key651 = iprot.readString();
+ _val652 = iprot.readDouble();
+ struct.shared_resources.put(_key651, _val652);
+ }
+ iprot.readMapEnd();
+ }
+ struct.set_shared_resources_isSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
@@ -721,6 +951,36 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
oprot.writeDouble(struct.shared_mem_off_heap);
oprot.writeFieldEnd();
}
+ if (struct.resources != null) {
+ if (struct.is_set_resources()) {
+ oprot.writeFieldBegin(RESOURCES_FIELD_DESC);
+ {
+ oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.DOUBLE, struct.resources.size()));
+ for (Map.Entry<String, Double> _iter654 : struct.resources.entrySet())
+ {
+ oprot.writeString(_iter654.getKey());
+ oprot.writeDouble(_iter654.getValue());
+ }
+ oprot.writeMapEnd();
+ }
+ oprot.writeFieldEnd();
+ }
+ }
+ if (struct.shared_resources != null) {
+ if (struct.is_set_shared_resources()) {
+ oprot.writeFieldBegin(SHARED_RESOURCES_FIELD_DESC);
+ {
+ oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.DOUBLE, struct.shared_resources.size()));
+ for (Map.Entry<String, Double> _iter655 : struct.shared_resources.entrySet())
+ {
+ oprot.writeString(_iter655.getKey());
+ oprot.writeDouble(_iter655.getValue());
+ }
+ oprot.writeMapEnd();
+ }
+ oprot.writeFieldEnd();
+ }
+ }
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -754,7 +1014,13 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
if (struct.is_set_shared_mem_off_heap()) {
optionals.set(4);
}
- oprot.writeBitSet(optionals, 5);
+ if (struct.is_set_resources()) {
+ optionals.set(5);
+ }
+ if (struct.is_set_shared_resources()) {
+ optionals.set(6);
+ }
+ oprot.writeBitSet(optionals, 7);
if (struct.is_set_mem_on_heap()) {
oprot.writeDouble(struct.mem_on_heap);
}
@@ -770,12 +1036,32 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
if (struct.is_set_shared_mem_off_heap()) {
oprot.writeDouble(struct.shared_mem_off_heap);
}
+ if (struct.is_set_resources()) {
+ {
+ oprot.writeI32(struct.resources.size());
+ for (Map.Entry<String, Double> _iter656 : struct.resources.entrySet())
+ {
+ oprot.writeString(_iter656.getKey());
+ oprot.writeDouble(_iter656.getValue());
+ }
+ }
+ }
+ if (struct.is_set_shared_resources()) {
+ {
+ oprot.writeI32(struct.shared_resources.size());
+ for (Map.Entry<String, Double> _iter657 : struct.shared_resources.entrySet())
+ {
+ oprot.writeString(_iter657.getKey());
+ oprot.writeDouble(_iter657.getValue());
+ }
+ }
+ }
}
@Override
public void read(org.apache.thrift.protocol.TProtocol prot, WorkerResources struct) throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(5);
+ BitSet incoming = iprot.readBitSet(7);
if (incoming.get(0)) {
struct.mem_on_heap = iprot.readDouble();
struct.set_mem_on_heap_isSet(true);
@@ -796,6 +1082,36 @@ public class WorkerResources implements org.apache.thrift.TBase<WorkerResources,
struct.shared_mem_off_heap = iprot.readDouble();
struct.set_shared_mem_off_heap_isSet(true);
}
+ if (incoming.get(5)) {
+ {
+ org.apache.thrift.protocol.TMap _map658 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.DOUBLE, iprot.readI32());
+ struct.resources = new HashMap<String,Double>(2*_map658.size);
+ String _key659;
+ double _val660;
+ for (int _i661 = 0; _i661 < _map658.size; ++_i661)
+ {
+ _key659 = iprot.readString();
+ _val660 = iprot.readDouble();
+ struct.resources.put(_key659, _val660);
+ }
+ }
+ struct.set_resources_isSet(true);
+ }
+ if (incoming.get(6)) {
+ {
+ org.apache.thrift.protocol.TMap _map662 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.DOUBLE, iprot.readI32());
+ struct.shared_resources = new HashMap<String,Double>(2*_map662.size);
+ String _key663;
+ double _val664;
+ for (int _i665 = 0; _i665 < _map662.size; ++_i665)
+ {
+ _key663 = iprot.readString();
+ _val664 = iprot.readDouble();
+ struct.shared_resources.put(_key663, _val664);
+ }
+ }
+ struct.set_shared_resources_isSet(true);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-client/src/jvm/org/apache/storm/scheduler/SupervisorDetails.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/scheduler/SupervisorDetails.java b/storm-client/src/jvm/org/apache/storm/scheduler/SupervisorDetails.java
index a0c94a8..0df8bdd 100644
--- a/storm-client/src/jvm/org/apache/storm/scheduler/SupervisorDetails.java
+++ b/storm-client/src/jvm/org/apache/storm/scheduler/SupervisorDetails.java
@@ -15,14 +15,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.scheduler;
import java.util.Collection;
import java.util.HashSet;
-import java.util.Set;
import java.util.Map;
+import java.util.Set;
-import org.apache.storm.Config;
+
+import org.apache.storm.Constants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,31 +33,31 @@ public class SupervisorDetails {
String id;
/**
- * hostname of this supervisor
+ * hostname of this supervisor.
*/
String host;
Object meta;
/**
- * meta data configured for this supervisor
+ * meta data configured for this supervisor.
*/
Object schedulerMeta;
/**
- * all the ports of the supervisor
+ * all the ports of the supervisor.
*/
Set<Integer> allPorts;
/**
- * Map containing a manifest of resources for the node the supervisor resides
+ * Map containing a manifest of resources for the node the supervisor resides.
*/
private Map<String, Double> _total_resources;
public SupervisorDetails(String id, String host, Object meta, Object schedulerMeta,
- Collection<? extends Number> allPorts, Map<String, Double> total_resources){
+ Collection<? extends Number> allPorts, Map<String, Double> total_resources) {
this.id = id;
this.host = host;
this.meta = meta;
this.schedulerMeta = schedulerMeta;
- if(allPorts!=null) {
+ if (allPorts != null) {
setAllPorts(allPorts);
} else {
this.allPorts = new HashSet<>();
@@ -72,7 +74,7 @@ public class SupervisorDetails {
this(id, null, meta, null, null, total_resources);
}
- public SupervisorDetails(String id, Object meta, Collection<? extends Number> allPorts){
+ public SupervisorDetails(String id, Object meta, Collection<? extends Number> allPorts) {
this(id, null, meta, null, allPorts, null);
}
@@ -87,13 +89,13 @@ public class SupervisorDetails {
@Override
public String toString() {
- return getClass().getSimpleName() + " ID: " + id + " HOST: " + host + " META: " + meta +
- " SCHED_META: " + schedulerMeta + " PORTS: " + allPorts;
+ return getClass().getSimpleName() + " ID: " + id + " HOST: " + host + " META: " + meta
+ + " SCHED_META: " + schedulerMeta + " PORTS: " + allPorts;
}
private void setAllPorts(Collection<? extends Number> allPorts) {
this.allPorts = new HashSet<>();
- if (allPorts!=null) {
+ if (allPorts != null) {
for (Number n: allPorts) {
this.allPorts.add(n.intValue());
}
@@ -125,14 +127,21 @@ public class SupervisorDetails {
}
public double getTotalMemory() {
- Double totalMemory = getTotalResource(Config.SUPERVISOR_MEMORY_CAPACITY_MB);
+ Double totalMemory = getTotalResource(Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME);
assert totalMemory != null;
return totalMemory;
}
public double getTotalCPU() {
- Double totalCPU = getTotalResource(Config.SUPERVISOR_CPU_CAPACITY);
+ Double totalCPU = getTotalResource(Constants.COMMON_CPU_RESOURCE_NAME);
assert totalCPU != null;
return totalCPU;
}
+
+ /**
+ * get all resources for this Supervisor.
+ */
+ public Map<String, Double> getTotalResources() {
+ return _total_resources;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-client/src/jvm/org/apache/storm/topology/BaseConfigurationDeclarer.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/topology/BaseConfigurationDeclarer.java b/storm-client/src/jvm/org/apache/storm/topology/BaseConfigurationDeclarer.java
index a0bb7ba..eac23bb 100644
--- a/storm-client/src/jvm/org/apache/storm/topology/BaseConfigurationDeclarer.java
+++ b/storm-client/src/jvm/org/apache/storm/topology/BaseConfigurationDeclarer.java
@@ -57,6 +57,7 @@ public abstract class BaseConfigurationDeclarer<T extends ComponentConfiguration
public T setMemoryLoad(Number onHeap) {
if (onHeap != null) {
onHeap = onHeap.doubleValue();
+ addResource(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, onHeap);
return addConfiguration(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, onHeap);
}
return (T) this;
@@ -70,6 +71,7 @@ public abstract class BaseConfigurationDeclarer<T extends ComponentConfiguration
if (offHeap!=null) {
offHeap = offHeap.doubleValue();
+ addResource(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, offHeap);
ret = addConfiguration(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, offHeap);
}
return ret;
@@ -78,9 +80,19 @@ public abstract class BaseConfigurationDeclarer<T extends ComponentConfiguration
@SuppressWarnings("unchecked")
@Override
public T setCPULoad(Number amount) {
- if(amount != null) {
+ if (amount != null) {
+ addResource(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, amount);
return addConfiguration(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, amount);
}
return (T) this;
}
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T addResources(Map<String, Double> resources) {
+ if (resources != null) {
+ return addConfiguration(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, resources);
+ }
+ return (T) this;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-client/src/jvm/org/apache/storm/topology/ComponentConfigurationDeclarer.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/topology/ComponentConfigurationDeclarer.java b/storm-client/src/jvm/org/apache/storm/topology/ComponentConfigurationDeclarer.java
index 19cbdab..8098029 100644
--- a/storm-client/src/jvm/org/apache/storm/topology/ComponentConfigurationDeclarer.java
+++ b/storm-client/src/jvm/org/apache/storm/topology/ComponentConfigurationDeclarer.java
@@ -15,20 +15,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.topology;
import java.util.Map;
public interface ComponentConfigurationDeclarer<T extends ComponentConfigurationDeclarer> extends ResourceDeclarer<T> {
/**
- * add in several configs to the component
+ * add in several configs to the component.
* @param conf the configs to add
* @return this for chaining.
*/
T addConfigurations(Map<String, Object> conf);
/**
- * Add in a single config
+ * return the configuration.
+ *
+ * @return the current configuration.
+ */
+ Map<String, Object> getRASConfiguration();
+
+ /**
+ * Add in a single config.
* @param config the key for the config
* @param value the value of the config
* @return this for chaining.
@@ -36,30 +44,40 @@ public interface ComponentConfigurationDeclarer<T extends ComponentConfiguration
T addConfiguration(String config, Object value);
/**
- * Turn on/off debug for this component
+ * Turn on/off debug for this component.
* @param debug true for debug on false for debug off
* @return this for chaining
*/
T setDebug(boolean debug);
/**
- * Set the max task parallelism for this component
+ * Set the max task parallelism for this component.
* @param val the maximum parallelism
* @return this for chaining
*/
T setMaxTaskParallelism(Number val);
/**
- * Set the max spout pending config for this component
+ * Set the max spout pending config for this component.
* @param val the value of max spout pending.
* @return this for chaining
*/
T setMaxSpoutPending(Number val);
/**
- * Set the number of tasks for this component
+ * Set the number of tasks for this component.
* @param val the number of tasks
* @return this for chaining.
*/
T setNumTasks(Number val);
+
+ /**
+ * Add generic resources for this component.
+ */
+ T addResources(Map<String, Double> resources);
+
+ /**
+ * Add generic resource for this component.
+ */
+ T addResource(String resourceName, Number resourceValue);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-client/src/jvm/org/apache/storm/topology/TopologyBuilder.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/topology/TopologyBuilder.java b/storm-client/src/jvm/org/apache/storm/topology/TopologyBuilder.java
index 971e5f3..0ebdae6 100644
--- a/storm-client/src/jvm/org/apache/storm/topology/TopologyBuilder.java
+++ b/storm-client/src/jvm/org/apache/storm/topology/TopologyBuilder.java
@@ -50,6 +50,8 @@ import org.json.simple.JSONValue;
import java.io.NotSerializableException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -58,6 +60,7 @@ import java.util.Set;
import static org.apache.storm.spout.CheckpointSpout.CHECKPOINT_COMPONENT_ID;
import static org.apache.storm.spout.CheckpointSpout.CHECKPOINT_STREAM_ID;
+import static org.apache.storm.utils.Utils.parseJson;
/**
* TopologyBuilder exposes the Java API for specifying a topology for Storm
@@ -576,10 +579,28 @@ public class TopologyBuilder {
throw new IllegalArgumentException("Cannot set serializations for a component using fluent API");
}
String currConf = _commons.get(_id).get_json_conf();
- _commons.get(_id).set_json_conf(mergeIntoJson(Utils.parseJson(currConf), conf));
+ _commons.get(_id).set_json_conf(mergeIntoJson(parseJson(currConf), conf));
return (T) this;
}
-
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T addResource(String resourceName, Number resourceValue) {
+ Map<String, Double> resourcesMap = (Map<String, Double>) getRASConfiguration().getOrDefault(
+ Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, new HashMap());
+
+ resourcesMap.put(resourceName, resourceValue.doubleValue());
+
+ getRASConfiguration().put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, resourcesMap);
+ return (T) this;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Map getRASConfiguration() {
+ return parseJson(_commons.get(_id).get_json_conf());
+ }
+
@SuppressWarnings("unchecked")
@Override
public T addSharedMemory(SharedMemory request) {
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-client/src/jvm/org/apache/storm/transactional/TransactionalTopologyBuilder.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/transactional/TransactionalTopologyBuilder.java b/storm-client/src/jvm/org/apache/storm/transactional/TransactionalTopologyBuilder.java
index 07d3524..f2deca8 100644
--- a/storm-client/src/jvm/org/apache/storm/transactional/TransactionalTopologyBuilder.java
+++ b/storm-client/src/jvm/org/apache/storm/transactional/TransactionalTopologyBuilder.java
@@ -15,9 +15,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.transactional;
-import org.apache.storm.coordination.IBatchBolt;
import org.apache.storm.coordination.BatchBoltExecutor;
import org.apache.storm.Config;
import org.apache.storm.Constants;
@@ -34,8 +34,9 @@ import org.apache.storm.topology.BaseConfigurationDeclarer;
import org.apache.storm.topology.BasicBoltExecutor;
import org.apache.storm.topology.BoltDeclarer;
import org.apache.storm.topology.IBasicBolt;
-import org.apache.storm.topology.IRichBolt;
+import org.apache.storm.coordination.IBatchBolt;
import org.apache.storm.topology.InputDeclarer;
+import org.apache.storm.topology.IRichBolt;
import org.apache.storm.topology.SpoutDeclarer;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.transactional.partitioned.IOpaquePartitionedTransactionalSpout;
@@ -228,10 +229,37 @@ public class TransactionalTopologyBuilder {
}
@Override
+ public Map getRASConfiguration() {
+ for(Map<String, Object> conf : _spoutConfs) {
+ if (conf.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP)) {
+ return conf;
+ }
+ }
+ Map<String, Object> newConf = new HashMap<>();
+ newConf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, new HashMap());
+ _spoutConfs.add(newConf);
+ return newConf;
+ }
+
+ @Override
public SpoutDeclarer addSharedMemory(SharedMemory request) {
_spoutSharedMemory.add(request);
return this;
- }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public SpoutDeclarer addResource(String resourceName, Number resourceValue) {
+ Map<String, Double> resourcesMap = (Map<String, Double>) getRASConfiguration().get(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP);
+
+ if (resourcesMap == null) {
+ resourcesMap = new HashMap<>();
+ }
+ resourcesMap.put(resourceName, resourceValue.doubleValue());
+
+ getRASConfiguration().put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, resourcesMap);
+ return this;
+ }
}
private static class BoltDeclarerImpl extends BaseConfigurationDeclarer<BoltDeclarer> implements BoltDeclarer {
@@ -534,9 +562,33 @@ public class TransactionalTopologyBuilder {
}
@Override
+ public Map getRASConfiguration() {
+ for(Map<String, Object> conf : _component.componentConfs) {
+ if (conf.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP)) {
+ return conf;
+ }
+ }
+ Map<String, Object> newConf = new HashMap<>();
+ newConf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, new HashMap());
+ _component.componentConfs.add(newConf);
+ return newConf;
+ }
+
+ @Override
public BoltDeclarer addSharedMemory(SharedMemory request) {
_component.sharedMemory.add(request);
return this;
}
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public BoltDeclarer addResource(String resourceName, Number resourceValue) {
+ Map<String, Double> resourcesMap = (Map<String, Double>) getRASConfiguration().get(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP);
+
+ resourcesMap.put(resourceName, resourceValue.doubleValue());
+
+ getRASConfiguration().put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, resourcesMap);
+ return this;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-client/src/jvm/org/apache/storm/trident/topology/TridentTopologyBuilder.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/trident/topology/TridentTopologyBuilder.java b/storm-client/src/jvm/org/apache/storm/trident/topology/TridentTopologyBuilder.java
index 3873117..be36b4e 100644
--- a/storm-client/src/jvm/org/apache/storm/trident/topology/TridentTopologyBuilder.java
+++ b/storm-client/src/jvm/org/apache/storm/trident/topology/TridentTopologyBuilder.java
@@ -371,6 +371,30 @@ public class TridentTopologyBuilder {
return this;
}
+ @SuppressWarnings("unchecked")
+ @Override
+ public SpoutDeclarer addResource(String resourceName, Number resourceValue) {
+ Map<String, Double> resourcesMap = (Map<String, Double>) getRASConfiguration().get(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP);
+
+ resourcesMap.put(resourceName, resourceValue.doubleValue());
+
+ getRASConfiguration().put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, resourcesMap);
+ return this;
+ }
+
+ @Override
+ public Map getRASConfiguration() {
+ for(Map<String, Object> conf : _component.componentConfs) {
+ if (conf.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP)) {
+ return conf;
+ }
+ }
+ Map<String, Object> newConf = new HashMap<>();
+ newConf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, new HashMap());
+ _component.componentConfs.add(newConf);
+ return newConf;
+ }
+
@Override
public SpoutDeclarer addSharedMemory(SharedMemory request) {
_component.sharedMemory.add(request);
@@ -763,9 +787,33 @@ public class TridentTopologyBuilder {
}
@Override
+ public Map getRASConfiguration() {
+ for(Map<String, Object> conf : _component.componentConfs) {
+ if (conf.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP)) {
+ return conf;
+ }
+ }
+ Map<String, Object> newConf = new HashMap<>();
+ newConf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, new HashMap());
+ _component.componentConfs.add(newConf);
+ return newConf;
+ }
+
+ @Override
public BoltDeclarer addSharedMemory(SharedMemory request) {
_component.sharedMemory.add(request);
return this;
}
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public BoltDeclarer addResource(String resourceName, Number resourceValue) {
+ Map<String, Double> resourcesMap = (Map<String, Double>) getRASConfiguration().get(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP);
+
+ resourcesMap.put(resourceName, resourceValue.doubleValue());
+
+ getRASConfiguration().put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, resourcesMap);
+ return this;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java b/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java
index 4a54ac7..1c8015a 100644
--- a/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java
+++ b/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java
@@ -18,25 +18,26 @@
package org.apache.storm.utils;
-import org.apache.commons.io.FileUtils;
-import org.apache.storm.Config;
-import org.apache.storm.daemon.supervisor.AdvancedFSOps;
-import org.apache.storm.generated.StormTopology;
-import org.apache.storm.validation.ConfigValidation;
-
import java.io.File;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
-import java.util.HashSet;
import java.util.concurrent.Callable;
import java.util.stream.Collectors;
+import org.apache.commons.io.FileUtils;
+import org.apache.storm.Config;
+import org.apache.storm.daemon.supervisor.AdvancedFSOps;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.validation.ConfigValidation;
+
+
public class ConfigUtils {
public static final String FILE_SEPARATOR = File.separator;
public final static String RESOURCES_SUBDIR = "resources";
@@ -79,7 +80,7 @@ public class ConfigUtils {
*/
public static Collection<String> readDirContents(String dir) {
Collection<File> ret = readDirFiles(dir);
- return ret.stream().map( car -> car.getName() ).collect( Collectors.toList() );
+ return ret.stream().map(car -> car.getName()).collect(Collectors.toList());
}
/**
@@ -433,5 +434,4 @@ public class ConfigUtils {
FileUtils.forceMkdir(new File(ret));
return ret;
}
-
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-client/src/jvm/org/apache/storm/utils/ObjectReader.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/utils/ObjectReader.java b/storm-client/src/jvm/org/apache/storm/utils/ObjectReader.java
index c6e340a..07fece1 100644
--- a/storm-client/src/jvm/org/apache/storm/utils/ObjectReader.java
+++ b/storm-client/src/jvm/org/apache/storm/utils/ObjectReader.java
@@ -91,7 +91,7 @@ public class ObjectReader {
if (o instanceof Number) {
return ((Number) o).doubleValue();
} else {
- throw new IllegalArgumentException("Don't know how to convert " + o + " to double");
+ throw new IllegalArgumentException("Don't know how to convert (" + o + ") to double");
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-client/src/storm.thrift
----------------------------------------------------------------------
diff --git a/storm-client/src/storm.thrift b/storm-client/src/storm.thrift
index a875e38..aff7507 100644
--- a/storm-client/src/storm.thrift
+++ b/storm-client/src/storm.thrift
@@ -496,6 +496,8 @@ struct WorkerResources {
3: optional double cpu;
4: optional double shared_mem_on_heap; //This is just for accounting mem_on_heap should be used for enforcement
5: optional double shared_mem_off_heap; //This is just for accounting mem_off_heap should be used for enforcement
+ 6: optional map<string, double> resources; // Generic resources Map
+ 7: optional map<string, double> shared_resources; // Shared Generic resources Map
}
struct Assignment {
1: required string master_code_dir;
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-server/src/main/java/org/apache/storm/LocalCluster.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/LocalCluster.java b/storm-server/src/main/java/org/apache/storm/LocalCluster.java
index fa49d12..de3053e 100644
--- a/storm-server/src/main/java/org/apache/storm/LocalCluster.java
+++ b/storm-server/src/main/java/org/apache/storm/LocalCluster.java
@@ -61,6 +61,8 @@ import org.apache.storm.generated.KeyNotFoundException;
import org.apache.storm.generated.KillOptions;
import org.apache.storm.generated.ListBlobsResult;
import org.apache.storm.generated.LogConfig;
+import org.apache.storm.generated.Nimbus.Iface;
+import org.apache.storm.generated.Nimbus.Processor;
import org.apache.storm.generated.NimbusSummary;
import org.apache.storm.generated.NotAliveException;
import org.apache.storm.generated.OwnerResourceSummary;
@@ -75,8 +77,6 @@ import org.apache.storm.generated.SupervisorPageInfo;
import org.apache.storm.generated.TopologyHistoryInfo;
import org.apache.storm.generated.TopologyInfo;
import org.apache.storm.generated.TopologyPageInfo;
-import org.apache.storm.generated.Nimbus.Iface;
-import org.apache.storm.generated.Nimbus.Processor;
import org.apache.storm.messaging.IContext;
import org.apache.storm.messaging.local.Context;
import org.apache.storm.nimbus.ILeaderElector;
@@ -93,12 +93,12 @@ import org.apache.storm.testing.TrackedTopology;
import org.apache.storm.utils.ConfigUtils;
import org.apache.storm.utils.DRPCClient;
import org.apache.storm.utils.NimbusClient;
-import org.apache.storm.utils.Utils;
import org.apache.storm.utils.ObjectReader;
import org.apache.storm.utils.RegisteredGlobalState;
import org.apache.storm.utils.StormCommonInstaller;
import org.apache.storm.utils.Time;
import org.apache.storm.utils.Time.SimulatedTime;
+import org.apache.storm.utils.Utils;
import org.apache.thrift.TException;
import org.json.simple.JSONValue;
import org.json.simple.parser.ParseException;
@@ -109,10 +109,10 @@ import org.slf4j.LoggerFactory;
* A stand alone storm cluster that runs inside a single process.
* It is intended to be used for testing. Both internal testing for
* Apache Storm itself and for people building storm topologies.
- *
+ *<p>
* LocalCluster is an AutoCloseable so if you are using it in tests you can use
* a try block to be sure it is shut down.
- *
+ * </p>
* try (LocalCluster cluster = new LocalCluster()) {
* // Do some tests
* }
@@ -159,7 +159,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
}
/**
- * Set the number of slots/ports each supervisor should have
+ * Set the number of slots/ports each supervisor should have.
*/
public Builder withPortsPerSupervisor(int portsPerSupervisor) {
if (portsPerSupervisor < 0) {
@@ -180,7 +180,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
}
/**
- * Add an single key/value config to the daemon conf
+ * Add an single key/value config to the daemon conf.
*/
public Builder withDaemonConf(String key, Object value) {
this.daemonConf.put(key, value);
@@ -330,6 +330,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
}
/**
+ * Builds a new LocalCluster.
* @return the LocalCluster
* @throws Exception on any one of many different errors.
* This is intended for testing so yes it is ugly and throws Exception...
@@ -340,6 +341,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
}
private static class TrackedStormCommon extends StormCommon {
+
private final String id;
public TrackedStormCommon(String id) {
this.id = id;
@@ -367,7 +369,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
private final SimulatedTime time;
/**
- * Create a default LocalCluster
+ * Create a default LocalCluster.
* @throws Exception on any error
*/
public LocalCluster() throws Exception {
@@ -375,7 +377,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
}
/**
- * Create a LocalCluster that connects to an existing Zookeeper instance
+ * Create a LocalCluster that connects to an existing Zookeeper instance.
* @param zkHost the host for ZK
* @param zkPort the port for ZK
* @throws Exception on any error
@@ -480,7 +482,13 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
}
}
}
-
+
+ /**
+ * Checks if Nimbuses have elected a leader.
+ * @return boolean
+ * @throws AuthorizationException
+ * @throws TException
+ */
private boolean hasLeader() throws AuthorizationException, TException {
ClusterSummary summary = getNimbus().getClusterInfo();
if (summary.is_set_nimbuses()) {
@@ -508,6 +516,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
}
public static final KillOptions KILL_NOW = new KillOptions();
+
static {
KILL_NOW.set_wait_secs(0);
}
@@ -920,7 +929,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
}
@Override
- public List<ProfileRequest> getComponentPendingProfileActions(String id, String component_id, ProfileAction action)
+ public List<ProfileRequest> getComponentPendingProfileActions(String id, String componentId, ProfileAction action)
throws TException {
// TODO Auto-generated method stub
throw new RuntimeException("NOT IMPLEMENTED YET");
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
index d9020e2..63a143d 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
@@ -54,6 +54,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.security.auth.Subject;
import org.apache.storm.Config;
+import org.apache.storm.Constants;
import org.apache.storm.DaemonConfig;
import org.apache.storm.StormTimer;
import org.apache.storm.blobstore.AtomicOutputStream;
@@ -2565,8 +2566,8 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
for (Map<String, Double> entry :
ResourceUtils.getBoltsResources(topology, topologyConf).values()) {
double memoryRequirement =
- entry.getOrDefault(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 0.0)
- + entry.getOrDefault(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 0.0);
+ entry.getOrDefault(Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, 0.0)
+ + entry.getOrDefault(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, 0.0);
if (memoryRequirement > largestMemoryOperator) {
largestMemoryOperator = memoryRequirement;
}
@@ -2574,8 +2575,8 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
for (Map<String, Double> entry :
ResourceUtils.getSpoutsResources(topology, topologyConf).values()) {
double memoryRequirement =
- entry.getOrDefault(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 0.0)
- + entry.getOrDefault(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 0.0);
+ entry.getOrDefault(Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, 0.0)
+ + entry.getOrDefault(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, 0.0);
if (memoryRequirement > largestMemoryOperator) {
largestMemoryOperator = memoryRequirement;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
index c49f50b..26fb729 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
@@ -26,10 +26,14 @@ import org.apache.storm.utils.ObjectReader;
import org.apache.storm.utils.Time;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static org.apache.storm.scheduler.resource.ResourceUtils.normalizedResourceMap;
+
public class SupervisorHeartbeat implements Runnable {
private final IStormClusterState stormClusterState;
@@ -73,11 +77,23 @@ public class SupervisorHeartbeat implements Runnable {
private Map<String, Double> mkSupervisorCapacities(Map<String, Object> conf) {
Map<String, Double> ret = new HashMap<String, Double>();
+ // Put in legacy values
Double mem = ObjectReader.getDouble(conf.get(Config.SUPERVISOR_MEMORY_CAPACITY_MB), 4096.0);
ret.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, mem);
Double cpu = ObjectReader.getDouble(conf.get(Config.SUPERVISOR_CPU_CAPACITY), 400.0);
ret.put(Config.SUPERVISOR_CPU_CAPACITY, cpu);
- return ret;
+
+
+ // If configs are present in Generic map and legacy - the legacy values will be overwritten
+ Map<String, Number> rawResourcesMap = (Map<String,Number>) conf.getOrDefault(
+ Config.SUPERVISOR_RESOURCES_MAP, Collections.emptyMap()
+ );
+
+ for (Map.Entry<String, Number> stringNumberEntry : rawResourcesMap.entrySet()) {
+ ret.put(stringNumberEntry.getKey(), stringNumberEntry.getValue().doubleValue());
+ }
+
+ return normalizedResourceMap(ret);
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/8a9141c2/storm-server/src/main/java/org/apache/storm/logging/ThriftAccessLogger.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/logging/ThriftAccessLogger.java b/storm-server/src/main/java/org/apache/storm/logging/ThriftAccessLogger.java
index 84540a8..e326174 100644
--- a/storm-server/src/main/java/org/apache/storm/logging/ThriftAccessLogger.java
+++ b/storm-server/src/main/java/org/apache/storm/logging/ThriftAccessLogger.java
@@ -15,7 +15,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.logging;
+
import java.net.InetAddress;
import java.security.Principal;
import org.slf4j.Logger;